From e028baa5dc5e052ccf188acd40d2474137b4bf7e Mon Sep 17 00:00:00 2001 From: Roham Koohestani Date: Sun, 6 Sep 2026 18:27:47 +0200 Subject: [PATCH 001/115] added agentic features from gitlab --- .env.example | 50 + pyproject.toml | 21 + src/Queries.py | 17 + src/agents/__init__.py | 17 + src/agents/event_writer.py | 226 ++ src/agents/inference.py | 557 +++ src/agents/ingest.py | 352 ++ src/agents/lifecycle.py | 77 + src/agents/normalize.py | 623 ++++ src/agents/provider.py | 117 + src/agents/registry.py | 63 + src/agents/telemetry.py | 129 + src/agents/tools.py | 88 + src/backend/Responses.py | 92 + src/backend/acp_authorization.py | 320 ++ src/backend/redis_manager.py | 30 + src/backend/routers/__init__.py | 16 +- src/backend/routers/acp/__init__.py | 360 ++ src/backend/routers/agent/__init__.py | 22 + src/backend/routers/agent/acp_auth.py | 66 + src/backend/routers/agent/consent.py | 113 + src/backend/routers/agent/ingest.py | 319 ++ src/backend/routers/agent/memory.py | 197 + src/backend/routers/agent/profiles.py | 418 +++ src/backend/routers/agents.py | 471 +++ src/backend/routers/analytics/studies.py | 329 +- src/backend/routers/session/deactivate.py | 17 + src/code4me2_agent/__init__.py | 16 + src/code4me2_agent/acp_runtime.py | 856 +++++ src/code4me2_agent/acp_updates.py | 199 + src/code4me2_agent/acp_utils.py | 9 + src/code4me2_agent/adapters.py | 1666 +++++++++ src/code4me2_agent/async_bridge.py | 56 + src/code4me2_agent/cli.py | 167 + src/code4me2_agent/command_tools.py | 692 ++++ src/code4me2_agent/config.py | 440 +++ src/code4me2_agent/echo.py | 176 + src/code4me2_agent/events.py | 45 + src/code4me2_agent/file_tools.py | 612 ++++ src/code4me2_agent/runtime_auth.py | 588 +++ src/code4me2_agent/telemetry.py | 219 ++ src/database/crud.py | 704 ++++ src/database/db_schemas.py | 397 ++ src/database/migration/migration_manager.py | 23 +- ...9cc1bc_baseline_migration_from_init_sql.py | 27 + .../versions/d1a2b3c4e5f7_add_agent_tables.py | 396 ++ ...2b3c4d5f6a8_seed_default_agent_profiles.py | 116 + .../EvaluationScript.py | 444 +-- .../EvaluationScriptChat.py | 462 +-- .../benchmark_results.csv | 3202 ++++++++--------- .../chat_benchmark_results.csv | 202 +- .../src/components/analytics/AgentResults.css | 112 + .../src/components/analytics/AgentResults.js | 301 ++ .../analytics/AnalyticsNavigation.js | 14 + .../components/analytics/StudyManagement.js | 37 +- src/website/src/pages/AgentAssignments.css | 244 ++ src/website/src/pages/AgentAssignments.js | 297 ++ src/website/src/pages/AgentProfiles.css | 316 ++ src/website/src/pages/AgentProfiles.js | 628 ++++ src/website/src/pages/Dashboard.js | 8 + src/website/src/utils/api.js | 133 + .../database_tests/test_migration_manager.py | 26 +- 62 files changed, 16761 insertions(+), 2176 deletions(-) create mode 100644 .env.example create mode 100644 src/agents/__init__.py create mode 100644 src/agents/event_writer.py create mode 100644 src/agents/inference.py create mode 100644 src/agents/ingest.py create mode 100644 src/agents/lifecycle.py create mode 100644 src/agents/normalize.py create mode 100644 src/agents/provider.py create mode 100644 src/agents/registry.py create mode 100644 src/agents/telemetry.py create mode 100644 src/agents/tools.py create mode 100644 src/backend/acp_authorization.py create mode 100644 src/backend/routers/acp/__init__.py create mode 100644 src/backend/routers/agent/__init__.py create mode 100644 src/backend/routers/agent/acp_auth.py create mode 100644 src/backend/routers/agent/consent.py create mode 100644 src/backend/routers/agent/ingest.py create mode 100644 src/backend/routers/agent/memory.py create mode 100644 src/backend/routers/agent/profiles.py create mode 100644 src/backend/routers/agents.py create mode 100644 src/code4me2_agent/__init__.py create mode 100644 src/code4me2_agent/acp_runtime.py create mode 100644 src/code4me2_agent/acp_updates.py create mode 100644 src/code4me2_agent/acp_utils.py create mode 100644 src/code4me2_agent/adapters.py create mode 100644 src/code4me2_agent/async_bridge.py create mode 100644 src/code4me2_agent/cli.py create mode 100644 src/code4me2_agent/command_tools.py create mode 100644 src/code4me2_agent/config.py create mode 100644 src/code4me2_agent/echo.py create mode 100644 src/code4me2_agent/events.py create mode 100644 src/code4me2_agent/file_tools.py create mode 100644 src/code4me2_agent/runtime_auth.py create mode 100644 src/code4me2_agent/telemetry.py create mode 100644 src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py create mode 100644 src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py create mode 100644 src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py create mode 100644 src/website/src/components/analytics/AgentResults.css create mode 100644 src/website/src/components/analytics/AgentResults.js create mode 100644 src/website/src/pages/AgentAssignments.css create mode 100644 src/website/src/pages/AgentAssignments.js create mode 100644 src/website/src/pages/AgentProfiles.css create mode 100644 src/website/src/pages/AgentProfiles.js diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c74eeaa --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# Example environment variables for the agent subsystem. +# +# PLACEHOLDER VALUES ONLY — never commit a real credential to this file. +# Copy the entries you need into your own `.env` (which is gitignored for real +# values) and fill them in there. +# +# This file documents only the agent-specific variables added by the agent +# subsystem. See README.md for the base platform's configuration (database, +# Redis, Celery, email, model serving). + + +# ─── Upstream LLM provider ─────────────────────────────────────────────────── +# +# Per merge decision 6 there is no hardcoded provider. Each `agent_profile` row +# names its own `base_url` + `api_key_ref` + `model`, and the only requirement +# is that the endpoint speaks the OpenAI-compatible chat-completions wire +# format. Ollama, Groq, OpenRouter, vLLM and OpenAI all qualify. +# +# Keys are NEVER stored in the database. A profile stores `api_key_ref`, the +# *name* of an environment variable, and the key is read from the environment +# at request time — so dumping the profile table cannot leak a credential. + +# Fallback upstream, used when a profile leaves `base_url` empty. +# Defaults to a local Ollama instance, which needs no key and costs nothing. +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 + +# Fallback API key, used when a profile leaves `api_key_ref` empty. +# Safe to leave blank for local providers that don't authenticate. +AGENT_UPSTREAM_API_KEY= + +# Per-provider keys. Create one variable per provider you use and point the +# matching profile's `api_key_ref` at its name. The seeded default profiles +# reference OLLAMA_API_KEY, AGENT_UPSTREAM_API_KEY and OPENAI_API_KEY. +OLLAMA_API_KEY= +OPENAI_API_KEY=your_openai_api_key_here +GROQ_API_KEY=your_groq_api_key_here +OPENROUTER_API_KEY=your_openrouter_api_key_here + + +# ─── Codex Responses API ───────────────────────────────────────────────────── +# +# Codex speaks OpenAI's proprietary Responses API rather than chat-completions, +# so it is handled as a special-cased normalization path. When a Codex profile +# targets a non-OpenAI base_url, the proprietary fields are rewritten; when it +# targets OpenAI, the payload is forwarded largely as-is. + +# Optional operator override for reasoning effort, e.g. to cap cost. +# Valid values: low | medium | high. Leave blank to let Codex decide (with a +# clamp applied for models that reject its default). +AGENT_REASONING_EFFORT= diff --git a/pyproject.toml b/pyproject.toml index e54d76c..00bac41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,24 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "code4me2-agent" +version = "0.1.0" +description = "Code4Me ACP agent runtime" +requires-python = ">=3.10" +dependencies = [ + "agent-client-protocol>=0.8", + "openai>=1.0,<2", +] + +[project.scripts] +code4me2-agent = "code4me2_agent.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["code4me2_agent*"] + [tool.ruff] line-length = 99 diff --git a/src/Queries.py b/src/Queries.py index 2fde242..28acf7b 100644 --- a/src/Queries.py +++ b/src/Queries.py @@ -384,6 +384,23 @@ class ActivateProject(QueryBase): project_id: UUID = Field(..., description="Project id to activate") +class PrepareAcpGrant(QueryBase): + """Plugin request for a one-time launch grant for a local agent process. + + Both fields are *bound into* the grant, so the agent that redeems it can + only ever act on this project and inside this workspace directory. + """ + + project_id: UUID = Field( + ..., description="Activated project id for the ACP process" + ) + workspace: str = Field(..., description="Canonical project workspace", min_length=1) + + +class ExchangeAcpGrant(QueryBase): + grant: str = Field(..., description="One-time ACP launch grant", min_length=1) + + class CreateSession(QueryBase): user_id: UUID = Field(..., description="User ID") diff --git a/src/agents/__init__.py b/src/agents/__init__.py new file mode 100644 index 0000000..07e0bb2 --- /dev/null +++ b/src/agents/__init__.py @@ -0,0 +1,17 @@ +"""Server-side agent subsystem. + +Layout: + +* ``provider`` — resolves the generic OpenAI-compatible upstream from a profile +* ``normalize`` — wire-format normalization (Chat Completions / Responses API) +* ``tools`` — catalogue of selectable tool names, per runtime +* ``telemetry`` — the per-inference-call record written by the proxy path +* ``event_writer`` — persistence for proxy-observed events +* ``ingest`` — translation + persistence for runtime-self-reported events +* ``inference`` — the authed relay behind POST /api/agent/inference +* ``registry`` — sticky, server-authoritative A/B profile assignment +* ``lifecycle`` — aggregating a finished task's events into totals + +Note this package is a *relay and telemetry sink*, not an agent runtime. The +built-in runtime lives in the separate ``code4me2_agent`` package. +""" diff --git a/src/agents/event_writer.py b/src/agents/event_writer.py new file mode 100644 index 0000000..b8c64fa --- /dev/null +++ b/src/agents/event_writer.py @@ -0,0 +1,226 @@ +"""Persistence of agent_event rows from the proxy telemetry path. + +Both telemetry paths converge on the same table (merge decision 2), so this +module and ``agents.ingest`` are two front doors onto one +``crud.append_agent_event``. The difference is only where the data comes from: +here it's observed by the relay; there it's self-reported by the runtime. + +Every write is best-effort: a telemetry failure must never break the agent the +developer is actually using, so DB errors are logged and swallowed. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime +from typing import TYPE_CHECKING, Optional + +from agents.normalize import snippet +from agents.telemetry import InferenceRecord +from database import crud + +if TYPE_CHECKING: + from App import App + +# Source tag written to agent_event.source for rows produced by this path. +SOURCE_PROXY = "proxy" + + +def write_model_call_event( + app: App, + task_uuid: Optional[uuid.UUID], + record: InferenceRecord, + latency_ms: int, + span: Optional[dict] = None, + extra: Optional[dict] = None, +) -> None: + """Persist one model_call event row. + + ``record`` carries the per-call fields; ``span`` carries the span/context + fields (span_id, parent_span_id, step_index, session detection state). + ``extra`` is an optional dict of non-content structural metrics with no + typed column, serialised into ``extra_json``. + + session_id / task_id are deliberately not stored on the event — they live on + ``agent_task``, and duplicating them here would let the two drift. + """ + if task_uuid is None: + return + span = span or {} + roles = record.role_breakdown or {} + parent_span_id = span.get("parent_span_id") + db = app.get_db_session() + try: + event_index = crud.count_agent_events_for_task(db, task_uuid) + crud.append_agent_event( + db, + task_id=task_uuid, + event_index=event_index, + event_type="model_call", + source=SOURCE_PROXY, + latency_ms=latency_ms, + # span identifiers + span_id=span.get("span_id") or record.request_id, + parent_span_id=_as_uuid(parent_span_id), + request_id=record.request_id, + chat_session_index=span.get("chat_session_index"), + # model_call metadata + model=record.model, + agent_profile=record.agent_profile, + streaming=record.streaming, + message_count=record.message_count, + role_system_count=roles.get("system"), + role_user_count=roles.get("user"), + role_assistant_count=roles.get("assistant"), + role_tool_count=roles.get("tool"), + tools_kept=record.tools_kept, + tools_stripped=record.tools_stripped, + tool_names_requested=record.tool_names_requested or None, + max_tokens=record.max_tokens, + prompt_tokens=record.prompt_tokens, + completion_tokens=record.completion_tokens, + total_tokens=record.total_tokens, + finish_reason=record.finish_reason, + upstream_status=record.upstream_status, + step_index=span.get("step_index"), + context_window_size_bytes=span.get("context_window_size_bytes"), + active_file=record.active_file, + first_message_hash=span.get("first_message_hash"), + chat_new_session_detected=span.get("chat_new_session_detected"), + experiment_tool_access_enabled=span.get("experiment_tool_access_enabled"), + experiment_approval_policy=span.get("experiment_approval_policy"), + # content — already nulled by the caller unless consent resolved True + first_system_message=record.first_system_message, + last_user_message=record.last_user_message, + response_text=record.response_text, + extra_json=json.dumps(extra, default=str) if extra else None, + ) + logging.info( + f"[Agent/events] model_call written — task={task_uuid} " + f"event_index={event_index} latency_ms={latency_ms} " + f"tokens={record.total_tokens}" + ) + except Exception as e: + logging.error( + f"[Agent/events] failed to write model_call for task={task_uuid} — {e}", + exc_info=True, + ) + finally: + db.close() + + +def write_tool_call_events( + app: App, + task_uuid: uuid.UUID, + chat_session_index: int, + tool_executions: list[dict], + prev_call_at: Optional[datetime] = None, + content_included: bool = False, + parent_span_id: Optional[str] = None, +) -> None: + """Persist one tool_call event per resolved tool execution. + + Third-party agents execute tools locally between inference calls, so there + are no per-tool timestamps available — only the gap between the previous + model_call and now. That gap is split evenly across the batch as an + approximate ``latency_ms``; it is an estimate, not a measurement. + + ``parent_span_id`` is the span_id of the model_call whose response requested + these tools, so each tool_call nests under it in the trace tree. Falls back + to the task root when unknown (e.g. the first call of a session). + + Tool arguments and results can contain whole file contents (a "view" result, + a patch), so they are only stored when ``content_included``; otherwise just + their lengths are recorded, which still supports "how much context did the + tool return" analysis without retaining the text. + """ + if not tool_executions: + return + + parent_span = _as_uuid(parent_span_id) or task_uuid + latency_ms: Optional[int] = None + if prev_call_at is not None: + elapsed_ms = (datetime.now() - prev_call_at).total_seconds() * 1000 + latency_ms = max(0, int(elapsed_ms / len(tool_executions))) + + db = app.get_db_session() + try: + event_index = crud.count_agent_events_for_task(db, task_uuid) + for offset, tc in enumerate(tool_executions): + arguments = tc.get("arguments") + result = tc.get("result") + tool_arguments: Optional[str] = None + tool_result: Optional[str] = None + tool_arguments_length: Optional[int] = None + tool_result_length: Optional[int] = None + if content_included: + tool_arguments = _as_text(arguments) + tool_result = _as_text(result) + # Lengths are structural, so record them either way — they're the + # only signal left when content storage is off. + if isinstance(arguments, str): + tool_arguments_length = len(arguments) + if isinstance(result, str): + tool_result_length = len(result) + + crud.append_agent_event( + db, + task_id=task_uuid, + event_index=event_index + offset, + event_type="tool_call", + source=SOURCE_PROXY, + latency_ms=latency_ms, + span_id=str(uuid.uuid4()), + parent_span_id=parent_span, + chat_session_index=chat_session_index, + tool_name=tc.get("name"), + tool_arguments=tool_arguments, + tool_result=tool_result, + tool_arguments_length=tool_arguments_length, + tool_result_length=tool_result_length, + commit=False, + ) + db.commit() + logging.info( + f"[Agent/events] recorded {len(tool_executions)} tool_call event(s) " + f"for task={task_uuid}" + ) + except Exception as e: + db.rollback() + logging.error( + f"[Agent/events] failed writing tool_call events for task={task_uuid} " + f"— {e}", + exc_info=True, + ) + finally: + db.close() + + +def _as_uuid(value: object) -> Optional[uuid.UUID]: + """Best-effort UUID coercion for span ids arriving as strings. + + Returns None rather than raising: an unparseable span id should cost the + parent link, not the whole telemetry row. + """ + if value is None: + return None + if isinstance(value, uuid.UUID): + return value + try: + return uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError): + return None + + +def _as_text(value: object) -> Optional[str]: + """Render a tool argument/result as a truncated string for a content column.""" + if value is None: + return None + if isinstance(value, str): + return snippet(value) + try: + return snippet(json.dumps(value, default=str)) + except (TypeError, ValueError): + return snippet(str(value)) diff --git a/src/agents/inference.py b/src/agents/inference.py new file mode 100644 index 0000000..5654848 --- /dev/null +++ b/src/agents/inference.py @@ -0,0 +1,557 @@ +"""Authed inference relay backing POST /api/agent/inference. + +This is the observation point for **third-party** agents (Goose, Codex). We +don't control their reasoning loops, so instead of asking them to report +telemetry, the plugin's local proxy transparently intercepts every inference +call they make and forwards it here (merge decision 2). That gives us: + +* per-call telemetry written into the shared ``agent_event`` table, identical in + shape to what the self-reporting ``code4me2-agent`` runtime produces; +* server-authoritative model / temperature / tool policy — the agent sends + whatever its own config says, and we override it from the profile snapshotted + onto the task, so an A/B arm actually takes effect; +* reconstruction of the tool calls the agent ran locally between requests, by + diffing the conversation against what we saw last time. + +The upstream is a *generic* OpenAI-compatible endpoint resolved from the task's +profile (merge decision 6) — Ollama, Groq, OpenRouter and OpenAI all work +through the same path. Codex's proprietary Responses API is the one special +case, handled as a normalization branch rather than the default. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +import uuid +from collections import Counter +from datetime import datetime +from typing import TYPE_CHECKING, Optional + +import httpx +from fastapi import Response +from fastapi.responses import StreamingResponse + +from agents import provider as provider_module +from agents.event_writer import write_model_call_event, write_tool_call_events +from agents.normalize import ( + extract_from_response, + extract_from_responses_api, + extract_tool_executions, + first_user_message_text, + is_meta_request, + normalize_responses_api_body, + parse_responses_api_stream, + parse_stream, + reconcile_openai_passthrough, + sanitize_schema, + snippet, + strip_info_msg, +) +from agents.telemetry import InferenceRecord +from agents.tools import KNOWN_AGENT_TOOLS +from database import crud + +if TYPE_CHECKING: + from App import App + +# Upstream request timeout. Agent turns with large contexts are slow, and a +# premature timeout looks to the developer like the agent hung. +_UPSTREAM_TIMEOUT_SECONDS = 120 + + +async def run_inference( + *, + task_uuid: uuid.UUID, + session_uuid: uuid.UUID, + openai_body: dict, + enrichment: Optional[dict], + agent_profile: Optional[str], + model: str, + temperature: Optional[float] = None, + base_url: Optional[str] = None, + api_key_ref: Optional[str] = None, + framework_version: Optional[str] = None, + content_included: bool = False, + profile_tools_json: Optional[str] = None, + app: App, +) -> Response: + """Forward one agent inference call upstream and record it as an agent_event. + + The task and session are pre-validated by the route handler, so this + function trusts its inputs — except ``content_included``, which the route + resolves from the user's stored preference and which is the only thing + gating whether message text is persisted. + + ``enrichment`` carries plugin-supplied IDE context (active file, observed + framework version). + """ + request_id = str(uuid.uuid4()) + enrichment = enrichment or {} + + # Which wire API this request uses is read off the body, not the profile: + # the Responses API (Codex) sends `input`, Chat Completions (Goose, and the + # built-in runtime) sends `messages`. Trusting the body means a + # misconfigured profile can't send a body down the wrong normalization path. + is_responses_api = "input" in openai_body + + upstream = provider_module.resolve_upstream( + model=model, + base_url=base_url, + api_key_ref=api_key_ref, + framework_version=framework_version, + ) + + # Codex → OpenAI is a passthrough: the Responses API is OpenAI's own, so + # when that's where we're sending it, none of the compat rewrites apply and + # the payload goes forward essentially untouched. Against any other + # provider the proprietary fields have to be normalized away. + is_openai_upstream = "api.openai.com" in upstream.base_url + use_openai_passthrough = is_responses_api and is_openai_upstream + + requested_model = openai_body.get("model", "unknown") + + if is_responses_api and not use_openai_passthrough: + logging.info( + "[Agent/inference] Responses API → non-OpenAI upstream, normalizing" + ) + normalize_responses_api_body(openai_body) + + # Override the model with the task's assigned one. The agent sends whatever + # its local config holds; the server decides authoritatively, from the + # profile snapshotted at task-creation time, so the A/B arm is real. + if model and model != requested_model: + logging.info( + f"[Agent/inference] model override — requested={requested_model!r} " + f"→ {model!r}" + ) + openai_body["model"] = model + elif not model: + # Defensive: task row had no model — fall back to the requested one so + # telemetry still has a concrete value to record. + model = requested_model + + if use_openai_passthrough: + reconcile_openai_passthrough(openai_body, model) + elif temperature is not None: + # Inject the profile's sampling temperature the same way as the model. + # Skipped on the OpenAI Responses passthrough, which doesn't accept it. + openai_body["temperature"] = temperature + logging.info(f"[Agent/inference] temperature override → {temperature}") + + streaming = bool(openai_body.get("stream", False)) + messages = openai_body.get("messages", []) or [] + max_tokens = openai_body.get("max_tokens") + # The active file path is structural metadata, stored unconditionally. + active_file: Optional[str] = enrichment.get("active_file") + observed_framework: Optional[str] = enrichment.get("framework_version") + + current_messages = openai_body.get("input", []) if is_responses_api else messages + if not isinstance(current_messages, list): + current_messages = [] + + # Count prior assistant turns to derive this call's step index. + if is_responses_api: + step_index = sum( + 1 + for item in current_messages + if isinstance(item, dict) and item.get("role") == "assistant" + ) + else: + step_index = sum( + 1 for m in messages if isinstance(m, dict) and m.get("role") == "assistant" + ) + + context_window_size_bytes = len( + json.dumps(current_messages, default=str).encode("utf-8") + ) + + # Detect whether this request belongs to a new chat session within the same + # task, by comparing a fingerprint of the first message against the previous + # request's. A *changed* first message means a brand-new chat was started; + # the same first message with a shorter array is more likely context + # compaction, which is not a new session. + message_count = len(current_messages) + first_message_hash: Optional[str] = ( + hashlib.sha256( + json.dumps(current_messages[0], default=str, sort_keys=True).encode("utf-8") + ).hexdigest() + if current_messages + else None + ) + chat_session_index = 0 + new_session_detected = False + prev_message_count = 0 + prev_call_at: Optional[datetime] = None + # span_id of the previous model_call — the parent of any tool calls the + # agent has executed since, because it was that response which requested them. + prev_model_call_span_id: Optional[str] = None + approval_policy: Optional[str] = None + meta_request = is_meta_request(current_messages) + + db_lookup = app.get_db_session() + try: + last_event = crud.get_last_agent_event_for_session( + db_lookup, session_uuid, event_type="model_call" + ) + if last_event: + prev_call_at = last_event.created_at + prev_model_call_span_id = last_event.span_id + prev_first_hash = last_event.first_message_hash + chat_session_index = last_event.chat_session_index or 0 + if meta_request: + # Auxiliary request (tool-call summary / chat-title generation): + # its message array is unrelated to the main chat, so pass the + # previous session-tracking state through unchanged rather than + # treating this as a new session — or letting it mask a real one + # on the next call. + first_message_hash = prev_first_hash + message_count = last_event.message_count or message_count + prev_message_count = message_count + elif prev_first_hash is not None and prev_first_hash != first_message_hash: + new_session_detected = True + chat_session_index += 1 + else: + prev_message_count = last_event.message_count or 0 + + task = crud.get_agent_task(db_lookup, task_uuid) + if task is not None: + approval_policy = task.approval_policy + # framework_version is a per-task property. Backfill the concrete + # version the proxy observed (e.g. "goose 1.x") over the profile's + # coarse runtime name, once, rather than on every event. + if observed_framework and not task.framework_version: + crud.set_agent_task_framework_version( + db_lookup, task_uuid, observed_framework + ) + task_description = task.task_description + # task_description is the user's first message — content, so only + # derive and persist it when the user opted into content storage. + if task_description is None and content_included: + first_user_text = first_user_message_text( + current_messages, is_responses_api=is_responses_api + ) + if first_user_text: + crud.set_agent_task_description( + db_lookup, task_uuid, snippet(first_user_text) + ) + except Exception as e: + logging.warning( + f"[Agent/inference] session-state lookup failed for task={task_uuid} — {e}" + ) + finally: + db_lookup.close() + + # Tool calls the agent executed locally appear, fully resolved, in the + # messages appended since our last call. Emit one tool_call event per + # call/result pair so we don't need a reporting hook inside the agent. + if 0 <= prev_message_count <= len(current_messages): + new_messages = current_messages[prev_message_count:] + else: + new_messages = current_messages + tool_executions = extract_tool_executions( + new_messages, is_responses_api=is_responses_api + ) + write_tool_call_events( + app, + task_uuid, + chat_session_index, + tool_executions, + prev_call_at, + content_included, + parent_span_id=prev_model_call_span_id, + ) + + system_content = next( + ( + m.get("content", "") + for m in messages + if isinstance(m, dict) and m.get("role") == "system" + ), + "", + ) + + original_tools = openai_body.get("tools", []) or [] + # Whether this condition has any tools available at all — distinguishes a + # deliberately tool-free arm from one with a non-empty allowlist. + experiment_tool_access_enabled = bool(original_tools) + tool_names_requested = [ + t.get("function", {}).get("name") + for t in original_tools + if isinstance(t, dict) and t.get("function", {}).get("name") + ] + tools_kept = 0 + tools_stripped = 0 + + if original_tools and not is_responses_api: + # Tool filtering and schema sanitisation apply only to Chat Completions. + # Codex manages its own tool schemas for the Responses API — those are + # passed through as-is (already normalized above where needed). + # + # Distinguish "profile explicitly selected no tools" (tools_json="[]" → + # empty set, must yield zero tools) from "profile has no tool selection + # at all" (null/unset → None, falls back to the global catalogue). This + # distinction is what makes a no-tools A/B arm possible. + profile_tools: Optional[set[str]] = None + if profile_tools_json is not None: + try: + parsed = json.loads(profile_tools_json) + if isinstance(parsed, list): + profile_tools = {str(t) for t in parsed} + except (json.JSONDecodeError, TypeError): + profile_tools = None + effective_allowlist = ( + profile_tools if profile_tools is not None else KNOWN_AGENT_TOOLS + ) + kept = [ + t + for t in original_tools + if t.get("function", {}).get("name") in effective_allowlist + ] + tools_stripped = len(original_tools) - len(kept) + tools_kept = len(kept) + for tool in kept: + sanitize_schema(tool.get("function", {}).get("parameters", {})) + if kept: + openai_body["tools"] = kept + else: + openai_body.pop("tools", None) + openai_body.pop("tool_choice", None) + logging.info( + f"[Agent/inference] tools — allowlist=" + f"{'profile' if profile_tools is not None else 'catalogue'} " + f"requested={tool_names_requested} kept={tools_kept} " + f"stripped={tools_stripped}" + ) + + # stream_options.include_usage is a Chat Completions extension and the only + # way to get token counts out of a stream; the Responses API reports usage + # in its completion event instead. + if streaming and not is_responses_api: + openai_body.setdefault("stream_options", {})["include_usage"] = True + + # Reasoning models add this to assistant messages, but non-reasoning models + # reject it with a 400 invalid_request_error. Since the server may override + # the model to a non-reasoning one, strip it defensively. + if "messages" in openai_body: + for msg in openai_body["messages"]: + if isinstance(msg, dict) and msg.get("role") == "assistant": + msg.pop("reasoning_content", None) + + body_bytes = json.dumps(openai_body).encode() + role_counts = Counter( + m.get("role", "unknown") for m in messages if isinstance(m, dict) + ) + last_user = next( + ( + m.get("content", "") + for m in reversed(messages) + if isinstance(m, dict) and m.get("role") == "user" + ), + None, + ) + last_user = ( + strip_info_msg(last_user) or None if isinstance(last_user, str) else None + ) + + # Conversation content is only persisted when the user opted in; otherwise + # these stay null and only the structural columns are written. + stored_system_message = (system_content or None) if content_included else None + stored_last_user = last_user if content_included else None + + span = { + "span_id": request_id, + "parent_span_id": str(task_uuid), + "step_index": step_index, + "context_window_size_bytes": context_window_size_bytes, + "first_message_hash": first_message_hash, + "chat_session_index": chat_session_index, + "chat_new_session_detected": new_session_detected, + "experiment_tool_access_enabled": experiment_tool_access_enabled, + "experiment_approval_policy": approval_policy, + } + # Non-content structural extras with no typed column of their own. + extra = { + "wire_api": "responses" if is_responses_api else "chat_completions", + "upstream_base_url": upstream.base_url, + "openai_passthrough": use_openai_passthrough, + "meta_request": meta_request, + "requested_model": requested_model, + } + + upstream_url = upstream.endpoint(responses_api=is_responses_api) + upstream_headers = { + "Authorization": f"Bearer {upstream.api_key}", + "Content-Type": "application/json", + } + + logging.info( + f"[Agent/inference] start request_id={request_id} task={task_uuid} " + f"model={model} stream={streaming} messages={len(messages)} " + f"max_tokens={max_tokens} → {upstream_url} body={len(body_bytes)}B" + ) + + t0 = time.monotonic() + + def _build_record( + prompt_tok: Optional[int], + completion_tok: Optional[int], + total_tok: Optional[int], + finish_reason: Optional[str], + response_text: Optional[str], + latency_ms: int, + upstream_status: int, + ) -> InferenceRecord: + return InferenceRecord( + request_id=request_id, + session_id=str(session_uuid), + task_id=str(task_uuid), + agent_profile=agent_profile, + model=model, + streaming=streaming, + message_count=len(messages), + role_breakdown=dict(role_counts), + first_system_message=stored_system_message, + last_user_message=stored_last_user, + tools_kept=tools_kept, + tools_stripped=tools_stripped, + tool_names_requested=tool_names_requested, + max_tokens=max_tokens, + active_file=active_file, + prompt_tokens=prompt_tok, + completion_tokens=completion_tok, + total_tokens=total_tok, + finish_reason=finish_reason, + response_text=response_text if content_included else None, + latency_ms=latency_ms, + upstream_status=upstream_status, + ) + + if streaming: + sse_buffer: list[bytes] = [] + captured_status: list[int] = [] + stream_aborted: list[bool] = [False] + + async def _stream(): + try: + async with httpx.AsyncClient( + timeout=_UPSTREAM_TIMEOUT_SECONDS + ) as client: + async with client.stream( + "POST", + upstream_url, + content=body_bytes, + headers=upstream_headers, + ) as upstream_resp: + captured_status.append(upstream_resp.status_code) + logging.info( + f"[Agent/inference] ← upstream " + f"status={upstream_resp.status_code} (stream)" + ) + if upstream_resp.status_code >= 400: + body = await upstream_resp.aread() + logging.error( + f"[Agent/inference] upstream error body: " + f"{body.decode('utf-8', errors='replace')}" + ) + sse_buffer.append(body) + yield body + return + async for chunk in upstream_resp.aiter_bytes(): + sse_buffer.append(chunk) + yield chunk + except Exception as e: + stream_aborted[0] = True + logging.warning(f"[Agent/inference] stream aborted mid-flight — {e}") + finally: + # Telemetry is written in `finally` so an aborted stream still + # produces a row — a dropped connection is itself a finding. + latency_ms = int((time.monotonic() - t0) * 1000) + raw_sse = b"".join(sse_buffer).decode("utf-8", errors="replace") + parser = parse_responses_api_stream if is_responses_api else parse_stream + ( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + ) = parser(raw_sse) + if stream_aborted[0]: + finish_reason = "stream_aborted" + + record = _build_record( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + latency_ms, + captured_status[0] if captured_status else 0, + ) + _log_record(record) + write_model_call_event( + app, task_uuid, record, latency_ms, span, extra + ) + + return StreamingResponse(_stream(), media_type="text/event-stream") + + async with httpx.AsyncClient(timeout=_UPSTREAM_TIMEOUT_SECONDS) as client: + upstream_resp = await client.post( + upstream_url, content=body_bytes, headers=upstream_headers + ) + + latency_ms = int((time.monotonic() - t0) * 1000) + + if upstream_resp.status_code >= 400: + logging.error( + f"[Agent/inference] upstream error {upstream_resp.status_code}: " + f"{upstream_resp.text}" + ) + + prompt_tok = completion_tok = total_tok = finish_reason = response_text = None + try: + extractor = ( + extract_from_responses_api if is_responses_api else extract_from_response + ) + prompt_tok, completion_tok, total_tok, finish_reason, response_text = extractor( + upstream_resp.json() + ) + except Exception as e: + # A body we can't parse costs telemetry detail, not the response — the + # agent still gets whatever the upstream said. + logging.warning(f"[Agent/inference] could not parse upstream response — {e}") + + record = _build_record( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + latency_ms, + upstream_resp.status_code, + ) + _log_record(record) + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + + return Response( + content=upstream_resp.content, + status_code=upstream_resp.status_code, + media_type="application/json", + ) + + +def _log_record(record: InferenceRecord) -> None: + status_flag = ( + "OK" if (record.upstream_status or 0) < 400 else f"ERR{record.upstream_status}" + ) + logging.info( + f"[Agent/inference] done request_id={record.request_id} " + f"task_id={record.task_id!r} {status_flag} model={record.model} " + f"stream={record.streaming} " + f"tokens(in/out/tot)={record.prompt_tokens}/{record.completion_tokens}/" + f"{record.total_tokens} finish={record.finish_reason} " + f"latency_ms={record.latency_ms} " + f"tools(kept/stripped)={record.tools_kept}/{record.tools_stripped}" + ) diff --git a/src/agents/ingest.py b/src/agents/ingest.py new file mode 100644 index 0000000..bedaa97 --- /dev/null +++ b/src/agents/ingest.py @@ -0,0 +1,352 @@ +"""Ingestion of self-reported telemetry from the built-in code4me2-agent runtime. + +This is the other half of merge decision 2 ("hybrid telemetry, per agent type"). +Because we *own* the ``code4me2-agent`` ReAct loop, it can report its own steps +rather than being observed from outside — which yields strictly better data than +proxy interception (it sees iterations, denials and parse failures that never +become HTTP requests). + +Both paths land in the same ``agent_event`` table, so downstream analysis never +has to care which runtime produced a row. This module is the translation layer: +it maps the runtime's event envelope onto the columnar schema, promoting known +fields to typed columns and routing the rest into the JSON overflow column. + +Two envelope quirks it reconciles: + +* The runtime reports a model call as *two* events (``agent.model.requested`` + then ``agent.model.completed``) and a tool call as two more + (``agent.tool.called`` / ``agent.tool.completed``), because it emits as it + goes. The proxy path writes one row per call. To keep the two paths + comparable, the request-side fields are merged forward onto the completion + row (matched by ``request_id`` / ``tool_call_id``), while the request events + are still kept as their own low-value rows so a crashed run isn't silently + lost. +* The envelope carries a client-side ``privacy`` block including + ``raw_capture_enabled``. That is **descriptive only** and is never used to + decide what gets stored — per merge decision 3 the consent check is + server-side (``resolve_store_agent_content``). The claimed block is retained + in ``extra_json`` purely so a mismatch between what the client thought and + what the server did is auditable. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from database import crud + +# Source tag written to agent_event.source for rows produced by this path. +SOURCE_SELF_REPORT = "code4me2_agent" + +# Runtime event_type → unified agent_event.event_type. +# +# `model_call` and `tool_call` are the two types that carry typed columns and +# feed task aggregation, so only the *completion* events map onto them; their +# request-side counterparts get distinct types to avoid double-counting tokens +# and steps in `lifecycle.finalize_agent_task`. +EVENT_TYPE_MAP: dict[str, str] = { + "agent.run.started": "run_started", + "agent.run.completed": "run_completed", + "agent.model.requested": "model_request", + "agent.model.completed": "model_call", + "agent.tool.called": "tool_request", + "agent.tool.completed": "tool_call", + "agent.tool.denied": "tool_denied", + "agent.tool.failed": "tool_failed", + "agent.adapter.loop_failed": "error", + "agent.adapter.parse_failed": "error", + "agent.request.received": "observation", + "agent.response.completed": "observation", +} + +# Metrics keys promoted out of the free-form `metrics` block into typed columns. +# Anything not listed stays in extra_json. +_PROMOTED_METRIC_KEYS = frozenset( + {"duration_ms", "prompt_tokens", "completion_tokens", "total_tokens"} +) + + +def map_event_type(runtime_event_type: str) -> str: + """Translate a runtime event type, defaulting unknown ones to `observation`. + + Unknown types are kept rather than rejected: the runtime evolves faster than + this map, and a new event name should degrade to a generic row with its + payload intact, not 400 the whole batch. + """ + mapped = EVENT_TYPE_MAP.get(runtime_event_type) + if mapped is None: + logging.info( + f"[Agent/ingest] unmapped event_type {runtime_event_type!r} " + f"— storing as 'observation'" + ) + return "observation" + return mapped + + +def _as_int(value: Any) -> Optional[int]: + """Coerce a reported value to int, or None if it isn't numeric.""" + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _as_uuid(value: Any) -> Optional[uuid.UUID]: + """Best-effort UUID coercion. The runtime mints ids as `uuid4().hex`, which + parses fine; anything else degrades to None rather than failing the row.""" + if value is None: + return None + try: + return uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError): + return None + + +def _parse_timestamp(value: Any) -> Optional[datetime]: + """Parse the runtime's ISO-8601 timestamp (it emits a trailing 'Z').""" + if isinstance(value, datetime): + return value + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _text_or_json(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + try: + return json.dumps(value, default=str) + except (TypeError, ValueError): + return str(value) + + +def build_correlation_index(events: list[dict]) -> tuple[dict, dict]: + """Pre-scan a batch to collect request-side fields for forward-merging. + + Returns ``(by_request_id, by_tool_call_id)``. Only fields that belong on the + completion row are collected, so the merge can't overwrite anything the + completion event reports itself. + """ + by_request_id: dict[str, dict] = {} + by_tool_call_id: dict[str, dict] = {} + + for event in events: + event_type = event.get("event_type") + payload = event.get("payload") or {} + if not isinstance(payload, dict): + continue + + if event_type == "agent.model.requested": + request_id = event.get("request_id") + if request_id: + by_request_id[str(request_id)] = { + "message_count": _as_int(payload.get("message_count")), + # The ReAct loop's iteration number is this call's step index. + "step_index": _as_int(payload.get("iteration")), + "approx_token_count": _as_int(payload.get("approx_token_count")), + } + elif event_type == "agent.tool.called": + tool_call_id = payload.get("tool_call_id") + if tool_call_id: + by_tool_call_id[str(tool_call_id)] = { + "arguments": payload.get("arguments"), + } + + return by_request_id, by_tool_call_id + + +def map_event_to_columns( + event: dict, + *, + content_included: bool, + by_request_id: dict, + by_tool_call_id: dict, +) -> dict: + """Translate one runtime event envelope into ``crud.append_agent_event`` kwargs. + + ``content_included`` is the server-resolved consent decision. When it is + False every content-bearing field (payload, raw_payload, tool arguments and + results, response text) is dropped here, before it can reach the database — + the structural columns and the non-content ``extra_json`` are unaffected. + """ + runtime_event_type = str(event.get("event_type") or "") + event_type = map_event_type(runtime_event_type) + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + metrics = event.get("metrics") if isinstance(event.get("metrics"), dict) else {} + request_id = event.get("request_id") + + # Forward-merge the request-side fields onto the completion row so a + # self-reported model_call has the same populated columns as a proxied one. + merged: dict = {} + if event_type == "model_call" and request_id: + merged = by_request_id.get(str(request_id), {}) + tool_arguments_value: Any = None + if event_type in ("tool_call", "tool_denied", "tool_failed"): + tool_call_id = payload.get("tool_call_id") + if tool_call_id: + tool_arguments_value = by_tool_call_id.get(str(tool_call_id), {}).get( + "arguments" + ) + + usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {} + + # latency: metrics.duration_ms is authoritative; tool events report it in + # their payload instead. + latency_ms = _as_int(metrics.get("duration_ms")) + if latency_ms is None: + latency_ms = _as_int(payload.get("duration_ms")) + + tool_result_value = payload.get("result") if "result" in payload else None + tool_arguments_text = _text_or_json(tool_arguments_value) + tool_result_text = _text_or_json(tool_result_value) + + # Non-content structural leftovers. `observability` and `privacy` are the + # runtime's own descriptive blocks; the un-promoted metrics keys are + # adapter-specific counters that don't warrant a column each. + extra: dict[str, Any] = { + "runtime_event_type": runtime_event_type, + "sequence": event.get("sequence"), + "message_id": event.get("message_id"), + } + leftover_metrics = { + k: v for k, v in metrics.items() if k not in _PROMOTED_METRIC_KEYS + } + if leftover_metrics: + extra["metrics"] = leftover_metrics + if isinstance(event.get("observability"), dict): + extra["observability"] = event["observability"] + if isinstance(event.get("privacy"), dict): + # Retained for audit only — the storage decision was made server-side. + extra["client_claimed_privacy"] = event["privacy"] + if merged.get("approx_token_count") is not None: + extra["approx_token_count"] = merged["approx_token_count"] + for key in ("status", "backend_type", "failure_reason", "denial_reason", "path"): + if payload.get(key) is not None: + extra[key] = payload[key] + + # Content: the payload may quote user prompts, model output or file + # contents, and raw_payload is the unredacted form of the same. Both are + # dropped wholesale without consent. + payload_json: Optional[str] = None + if content_included: + content_blob: dict[str, Any] = {"payload": payload} + if event.get("raw_payload") is not None: + content_blob["raw_payload"] = event["raw_payload"] + payload_json = _text_or_json(content_blob) + + return { + "event_type": event_type, + "source": SOURCE_SELF_REPORT, + "schema_version": event.get("schema_version"), + "occurred_at": _parse_timestamp(event.get("timestamp")), + "latency_ms": latency_ms, + # The runtime's own event id becomes the span id, which is what makes + # re-ingestion of a retried batch detectable. + "span_id": str(event["event_id"]) if event.get("event_id") else None, + "parent_span_id": _as_uuid(event.get("parent_event_id")), + "request_id": str(request_id) if request_id else None, + # model_call fields + "model": payload.get("model"), + "message_count": merged.get("message_count"), + "step_index": merged.get("step_index"), + "prompt_tokens": _as_int( + metrics.get("prompt_tokens") or usage.get("prompt_tokens") + ), + "completion_tokens": _as_int( + metrics.get("completion_tokens") or usage.get("completion_tokens") + ), + "total_tokens": _as_int( + metrics.get("total_tokens") or usage.get("total_tokens") + ), + "finish_reason": payload.get("finish_reason"), + # tool_call fields + "tool_name": payload.get("tool_name"), + "tool_arguments": tool_arguments_text if content_included else None, + "tool_result": tool_result_text if content_included else None, + # Lengths are structural — recorded regardless of consent, so "how much + # context did this tool return" stays answerable either way. + "tool_arguments_length": ( + len(tool_arguments_text) if tool_arguments_text is not None else None + ), + "tool_result_length": ( + len(tool_result_text) if tool_result_text is not None else None + ), + # JSON overflow + "extra_json": json.dumps(extra, default=str), + "payload_json": payload_json, + } + + +def ingest_event_batch( + db: Session, + *, + task_id: uuid.UUID, + events: list[dict], + content_included: bool, + agent_profile: Optional[str] = None, +) -> tuple[int, list[str]]: + """Persist a batch of self-reported events against ``task_id``. + + Returns ``(ingested_count, skipped_span_ids)``. Already-seen event ids are + skipped rather than erroring, which makes the endpoint safe for the runtime + to retry after a network failure — the uploader has no way to know whether a + timed-out POST was applied. + + The whole batch commits once: a partial write would leave gaps in + ``event_index``, which downstream step counting relies on being dense. + """ + if not events: + return 0, [] + + # Sort by the runtime's own sequence so event_index reflects real ordering + # even if the batch arrives out of order. + ordered = sorted(events, key=lambda e: e.get("sequence") or 0) + + span_ids = [str(e["event_id"]) for e in ordered if e.get("event_id")] + already_seen = set(crud.find_existing_agent_event_span_ids(db, span_ids)) + + by_request_id, by_tool_call_id = build_correlation_index(ordered) + next_index = crud.count_agent_events_for_task(db, task_id) + + ingested = 0 + skipped: list[str] = [] + for event in ordered: + span_id = str(event["event_id"]) if event.get("event_id") else None + if span_id is not None and span_id in already_seen: + skipped.append(span_id) + continue + columns = map_event_to_columns( + event, + content_included=content_included, + by_request_id=by_request_id, + by_tool_call_id=by_tool_call_id, + ) + crud.append_agent_event( + db, + task_id=task_id, + event_index=next_index + ingested, + agent_profile=agent_profile, + commit=False, + **columns, + ) + ingested += 1 + + db.commit() + logging.info( + f"[Agent/ingest] task={task_id} ingested={ingested} " + f"skipped_duplicates={len(skipped)} content={content_included}" + ) + return ingested, skipped diff --git a/src/agents/lifecycle.py b/src/agents/lifecycle.py new file mode 100644 index 0000000..19a9c53 --- /dev/null +++ b/src/agents/lifecycle.py @@ -0,0 +1,77 @@ +"""Finalizing an AgentTask: aggregating its events into task-level totals. + +Called from three places: + +* ``POST /api/agent/task/{task_id}/close`` — the plugin closes the previous + session's task on startup, because it cannot reliably detect when an ACP + agent process exits; +* session deactivation — closes any tasks left open when the user logs out; +* ``POST /api/agent/events/ingest`` — when a self-reporting runtime sends its + ``run_completed`` event. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy.orm import Session + +from database import crud + + +def finalize_agent_task( + db: Session, task_id: uuid.UUID, status: str = "done" +) -> dict: + """Aggregate a task's events into task-level totals and mark it terminal. + + Safe to call multiple times: totals are recomputed from the same events, so + re-running is idempotent regardless of the task's current status. That + matters because the plugin's close-on-startup call and session deactivation + can both fire for the same task. + """ + events = crud.get_agent_events_by_task(db, task_id) + model_call_events = [e for e in events if e.event_type == "model_call"] + + total_prompt = 0 + total_completion = 0 + agent_profile: Optional[str] = None + tool_names: Optional[list[str]] = None + for event in model_call_events: + total_prompt += event.prompt_tokens or 0 + total_completion += event.completion_tokens or 0 + if agent_profile is None and event.agent_profile: + agent_profile = event.agent_profile + if tool_names is None and event.tool_names_requested: + tool_names = list(event.tool_names_requested) + + last_event = max(events, key=lambda e: e.event_index) if events else None + total_steps = (last_event.event_index + 1) if last_event is not None else 0 + crud.update_agent_task_status( + db, + task_id=task_id, + status=status, + completed_at=datetime.now(), + total_steps=total_steps, + input_tokens=total_prompt or None, + output_tokens=total_completion or None, + latest_event_id=last_event.event_id if last_event else None, + ) + # Record the tools the agent actually had available, which for third-party + # runtimes is only knowable after observing a call — the profile's + # tools_json is our *request*, this is the observed reality. + if tool_names is not None: + crud.update_agent_task_tools(db, task_id=task_id, tools=tool_names) + + summary = { + "status": status, + "total_steps": total_steps, + "prompt_tokens": total_prompt, + "completion_tokens": total_completion, + "agent_profile": agent_profile, + "tools": tool_names, + } + logging.info(f"[Agent/lifecycle] task {task_id} finalized — {summary}") + return summary diff --git a/src/agents/normalize.py b/src/agents/normalize.py new file mode 100644 index 0000000..8d55532 --- /dev/null +++ b/src/agents/normalize.py @@ -0,0 +1,623 @@ +"""Wire-format normalization for the agent inference relay. + +Third-party agents don't speak one dialect. This module holds everything needed +to turn what they actually send into something a generic OpenAI-compatible +upstream will accept, and to read telemetry back out of the response: + +* **Chat Completions** (``messages``) — what Goose and the built-in + ``code4me2-agent`` runtime send. Needs JSON-Schema sanitisation, because + Goose's Rust-generated tool schemas carry keywords (``$ref``, ``uint32`` + formats, union ``type`` arrays) that most providers reject outright. +* **Responses API** (``input``) — what Codex sends. OpenAI-proprietary and *not* + chat-completions-shaped, so per merge decision 6 it stays a special-cased + path rather than the default: when the target isn't OpenAI itself, the + proprietary fields have to be stripped or rewritten. + +Also here: extracting token counts / finish reason / response text from both +streaming (SSE) and non-streaming replies, and reconstructing the tool calls an +agent executed locally between two inference requests. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Optional + +_SNIPPET_LEN = 300 + + +def snippet(text: str) -> str: + """Collapse text to a single short line for logs and content columns.""" + text = text.strip().replace("\n", " ") + return text[:_SNIPPET_LEN] + "…" if len(text) > _SNIPPET_LEN else text + + +# ── Goose-specific prompt cruft ─────────────────────────────────────────────── + +# Goose prepends an ... block (current time, working +# directory, todo notes) to every user message it sends — strip it so telemetry +# reflects the user's actual request, not this injected context. +_INFO_MSG_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) + +# Goose also sends separate "generate a chat title" requests, which wrap the +# real user message(s) like: +# ---BEGIN USER MESSAGES--- +# +# ---END USER MESSAGES--- +# Generate a short title for the above messages. +# Unwrap these so telemetry captures the underlying request, not the title prompt. +_TITLE_GEN_RE = re.compile( + r"---BEGIN USER MESSAGES---(.*?)---END USER MESSAGES---", re.DOTALL +) + + +def strip_info_msg(text: str) -> str: + text = _INFO_MSG_RE.sub("", text) + title_match = _TITLE_GEN_RE.search(text) + if title_match: + text = title_match.group(1) + return text.strip() + + +# Goose fires small auxiliary requests alongside the main conversation — e.g. a +# "summarize this tool call" request after each tool execution, and the +# chat-title request matched by _TITLE_GEN_RE above. These have their own tiny, +# unrelated message arrays and must not be mistaken for (or mask) a real new +# chat session. +_META_SYSTEM_SUBSTRINGS = ("summarize this tool call",) + + +def is_meta_request(current_messages: list) -> bool: + if not current_messages: + return False + for msg in current_messages: + if not isinstance(msg, dict): + continue + content = msg.get("content", "") + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + if not isinstance(content, str): + continue + content_lower = content.lower() + if any(p in content_lower for p in _META_SYSTEM_SUBSTRINGS): + return True + if "---begin user messages---" in content_lower: + return True + return False + + +def first_user_message_text( + messages: list, is_responses_api: bool = False +) -> Optional[str]: + """Find the first user-role message's text — used as the task description on + the first inference call of a task, when none was supplied by the plugin.""" + for m in messages: + if not isinstance(m, dict): + continue + if is_responses_api: + if m.get("type") != "message" or m.get("role") != "user": + continue + content = m.get("content", "") + text = ( + " ".join( + c.get("text", "") + for c in content + if isinstance(c, dict) and c.get("type") == "input_text" + ) + if isinstance(content, list) + else str(content) + ) + else: + if m.get("role") != "user": + continue + content = m.get("content", "") + text = content if isinstance(content, str) else "" + text = strip_info_msg(text) + if text: + return text + return None + + +def extract_tool_executions( + messages: list, is_responses_api: bool = False +) -> list[dict]: + """Pair tool-call requests with their results from newly-appended messages. + + Third-party agents execute tools locally and append both the call (assistant + ``tool_calls``) and its result (a ``tool``-role message, matched by + ``tool_call_id``) to the conversation before the next inference request: + + {"role": "assistant", "tool_calls": [{"id": "fc_...", + "function": {"name": "tree", "arguments": "{\\"path\\": \\".\\"}"}}]} + {"role": "tool", "tool_call_id": "fc_...", "content": "src/ [85K]\\n..."} + + That's what lets the proxy record tool_call events without needing a + reporting hook from inside the agent. The Responses API + (``function_call``/``function_call_output``) branch follows the same + item-list shape. Calls without a matching result yet are reported with a + None result. + """ + pending: dict[str, dict] = {} + completed: list[dict] = [] + + if is_responses_api: + for item in messages: + if not isinstance(item, dict): + continue + if item.get("type") == "function_call": + call_id = item.get("call_id") + if call_id: + pending[call_id] = { + "name": item.get("name"), + "arguments": item.get("arguments"), + } + elif item.get("type") == "function_call_output": + call = pending.pop(item.get("call_id"), None) + output = item.get("output") + if isinstance(output, list): + output = " ".join( + c.get("text", "") for c in output if isinstance(c, dict) + ) + completed.append( + { + "name": call.get("name") if call else None, + "arguments": call.get("arguments") if call else None, + "result": output, + } + ) + else: + for m in messages: + if not isinstance(m, dict): + continue + if m.get("role") == "assistant" and isinstance(m.get("tool_calls"), list): + for tc in m["tool_calls"]: + if not isinstance(tc, dict): + continue + call_id = tc.get("id") + fn = tc.get("function") or {} + if call_id: + pending[call_id] = { + "name": fn.get("name"), + "arguments": fn.get("arguments"), + } + elif m.get("role") == "tool": + call = pending.pop(m.get("tool_call_id"), None) + content = m.get("content") + if isinstance(content, list): + content = " ".join( + c.get("text", "") for c in content if isinstance(c, dict) + ) + completed.append( + { + "name": call.get("name") if call else None, + "arguments": call.get("arguments") if call else None, + "result": content, + } + ) + + # Calls still awaiting a result (e.g. tool failed before producing output). + for call in pending.values(): + completed.append( + { + "name": call.get("name"), + "arguments": call.get("arguments"), + "result": None, + } + ) + + return completed + + +# ── JSON Schema sanitisation ───────────────────────────────────────────────── +# +# Fields that OpenAI-compatible tool-calling APIs do not support. +# +# References: +# OpenAI function calling: https://platform.openai.com/docs/guides/function-calling +# OpenAI structured outputs: https://platform.openai.com/docs/guides/structured-outputs#supported-schemas +# Groq tool use: https://console.groq.com/docs/tool-use +# +# Supported subset (keep these): +# type (single string), description, properties, required, enum, items, +# anyOf, minimum/maximum, minLength/maxLength, minItems/maxItems, const, +# default, pattern +_STRIP_KEYS = frozenset( + { + # JSON Schema meta / reference keywords — not part of the OpenAI subset + # https://json-schema.org/understanding-json-schema/structuring + "$schema", + "$ref", + "$defs", + "$id", + "$anchor", + # "title" is valid JSON Schema but several providers reject it at the + # parameter level + "title", + # Explicitly unsupported by OpenAI structured outputs, and a common + # cause of provider-side failed_generation errors: + # https://platform.openai.com/docs/guides/structured-outputs#some-type-specific-keywords-are-not-yet-supported + "additionalProperties", + "unevaluatedProperties", + "patternProperties", + "propertyNames", + "minProperties", + "maxProperties", + # Conditional composition keywords — not in the supported subset + "if", + "then", + "else", + "not", + # Deprecated OpenAI nullable flag; use anyOf: [{type: T}, {type: "null"}] + # https://platform.openai.com/docs/guides/structured-outputs#all-fields-must-be-required + "nullable", + } +) + +# Non-standard numeric format strings providers reject (Goose's Rust-generated +# schemas emit these). The OpenAI spec only recognises "date-time", "email", +# "uri", etc. — not Rust integer widths. +_UNSUPPORTED_FORMATS = frozenset( + {"uint8", "uint16", "uint32", "uint64", "int32", "int64"} +) + + +def _inline_refs(params: dict, defs: dict) -> None: + """Resolve $ref pointers using the $defs map, inlining the definition in-place. + + Only handles local fragment refs of the form "#/$defs/Name". Unknown refs are + left as-is and will be stripped by ``sanitize_schema`` afterwards. + """ + if "$ref" in params: + ref = params.pop("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + name = ref[len("#/$defs/") :] + resolved = defs.get(name) + if isinstance(resolved, dict): + params.update({k: v for k, v in resolved.items() if k not in params}) + + for child in params.get("properties", {}).values(): + if isinstance(child, dict): + _inline_refs(child, defs) + + for schema_list_key in ("anyOf", "oneOf", "allOf"): + for child in params.get(schema_list_key, []): + if isinstance(child, dict): + _inline_refs(child, defs) + + items = params.get("items") + if isinstance(items, dict): + _inline_refs(items, defs) + + +def sanitize_schema(params: dict) -> None: + """Resolve $ref/$defs, then recursively strip JSON Schema fields providers reject. + + Mutates ``params`` in place. Supported subset: type (string), description, + properties, required, enum, items, anyOf, minimum/maximum, + minLength/maxLength, minItems/maxItems, const, default, pattern. + """ + # Inline $ref references before stripping $defs so no information is lost — + # e.g. Goose's extensionmanager__manage_extensions.action is a $ref to an + # enum in $defs, and dropping it unresolved would lose the allowed values. + defs = params.get("$defs", {}) + if defs: + _inline_refs(params, defs) + + for key in _STRIP_KEYS: + params.pop(key, None) + + fmt = params.get("format") + if isinstance(fmt, str) and fmt in _UNSUPPORTED_FORMATS: + params.pop("format") + + if isinstance(params.get("type"), list): + types = params["type"] + non_null = [t for t in types if t != "null"] + if len(non_null) == 1: + params["type"] = non_null[0] + if params.get("default") is None and "default" in params: + params.pop("default") + + for child in params.get("properties", {}).values(): + if isinstance(child, dict): + sanitize_schema(child) + + items = params.get("items") + if isinstance(items, dict): + sanitize_schema(items) + + for schema_list_key in ("anyOf", "oneOf", "allOf"): + for child in params.get(schema_list_key, []): + if isinstance(child, dict): + sanitize_schema(child) + + +# ── Responses API (Codex) normalization ────────────────────────────────────── + +# Top-level Responses API fields that are OpenAI-proprietary and rejected by +# other providers. Codex always sends these; strip them before forwarding to a +# non-OpenAI upstream. +_RESPONSES_API_STRIP_KEYS = frozenset( + { + "reasoning", # o-series reasoning effort config + "include", # e.g. ["reasoning.encrypted_content"] + "text", # verbosity / format config + "prompt_cache_key", # OpenAI prompt caching + "store", # OpenAI storage flag + "client_metadata", # Codex internal telemetry + "service_tier", # OpenAI tier routing (e.g. "priority") + "parallel_tool_calls", # Chat Completions concept — not in the Responses spec + # Note: `instructions` is handled separately below — it's injected as a + # system message into the input array rather than simply dropped. + } +) + +_VALID_REASONING_EFFORTS = frozenset({"low", "medium", "high"}) + + +def normalize_responses_api_body(body: dict) -> None: + """Rewrite a Codex Responses-API body so a generic provider will accept it. + + Only called when the upstream is *not* OpenAI (see + ``reconcile_openai_passthrough`` for that case). Mutates ``body`` in place: + + * drops OpenAI-proprietary top-level fields; + * moves ``instructions`` into the input array as a system message, since + it's an OpenAI-only top-level field but the content still matters; + * rewrites the OpenAI-only ``developer`` role to ``system``; + * coerces every tool to ``type="function"``. Codex also sends + ``type="custom"`` freeform-grammar tools (apply_patch, exec_command, …) + and hosted tools (web_search, …), which generic providers don't + implement — exposing them as plain functions with a single ``input`` + string keeps them callable, and Codex still executes the result locally. + """ + for key in _RESPONSES_API_STRIP_KEYS: + body.pop(key, None) + + instructions = body.pop("instructions", None) + if instructions: + input_items = body.get("input", []) + if isinstance(input_items, list): + input_items.insert( + 0, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": instructions}], + }, + ) + body["input"] = input_items + + for item in body.get("input", []) if isinstance(body.get("input"), list) else []: + if isinstance(item, dict) and item.get("role") == "developer": + item["role"] = "system" + + raw_tools = body.get("tools") + if isinstance(raw_tools, list): + sanitized = [] + for t in raw_tools: + if not isinstance(t, dict) or not t.get("name"): + continue # tools without a name are unusable — drop silently + if t.get("type") == "function": + params = t.get("parameters", {}) + sanitize_schema(params) + sanitized.append( + { + "type": "function", + "name": t["name"], + "description": t.get("description", ""), + "parameters": params, + } + ) + else: + sanitized.append( + { + "type": "function", + "name": t["name"], + "description": t.get("description", ""), + "parameters": { + "type": "object", + "properties": {"input": {"type": "string"}}, + "required": ["input"], + }, + } + ) + body["tools"] = sanitized + body.pop("tool_choice", None) + + +def reconcile_openai_passthrough(body: dict, model: str) -> None: + """Fix up a Responses-API body that *is* going to OpenAI, for a pinned model. + + When the server overrides Codex's requested model, parameters Codex chose + for its original model can become invalid for ours, 400-ing an otherwise + fine request. Reconcile the two known cases: + + * ``gpt-5.1-codex-mini`` only accepts ``text.verbosity == "medium"``, + while Codex defaults to ``"low"`` for other models; + * codex-mini only accepts low/medium/high reasoning effort, so Codex's + ``xhigh`` has to be clamped. + + ``AGENT_REASONING_EFFORT`` is an explicit operator override (e.g. to cap + cost) and takes precedence over the clamp. + """ + import os + + if "codex-mini" in model: + text_cfg = body.get("text") + if isinstance(text_cfg, dict) and text_cfg.get("verbosity") not in ( + None, + "medium", + ): + logging.info( + f"[Agent/normalize] coercing text.verbosity=" + f"{text_cfg.get('verbosity')!r} → 'medium' for {model!r}" + ) + text_cfg["verbosity"] = "medium" + + reasoning_cfg = body.get("reasoning") + if not isinstance(reasoning_cfg, dict): + return + + effort_override = os.getenv("AGENT_REASONING_EFFORT", "").strip() + if effort_override: + if reasoning_cfg.get("effort") != effort_override: + logging.info( + f"[Agent/normalize] reasoning.effort override via env — " + f"{reasoning_cfg.get('effort')!r} → {effort_override!r}" + ) + reasoning_cfg["effort"] = effort_override + return + + if "codex-mini" in model: + effort = reasoning_cfg.get("effort") + if isinstance(effort, str) and effort not in _VALID_REASONING_EFFORTS: + logging.info( + f"[Agent/normalize] coercing reasoning.effort={effort!r} → 'high' " + f"for {model!r}" + ) + reasoning_cfg["effort"] = "high" + + +# ── Response parsing ───────────────────────────────────────────────────────── + +_ParsedResponse = tuple[ + Optional[int], Optional[int], Optional[int], Optional[str], Optional[str] +] + + +def parse_stream(raw_sse: str) -> _ParsedResponse: + """Extract (prompt_tokens, completion_tokens, total_tokens, finish_reason, + response_text) from a Chat Completions SSE stream. + + Token counts only appear when the request set + ``stream_options.include_usage``; otherwise they come back None. + """ + content_parts: list[str] = [] + finish_reason = None + prompt_tokens = completion_tokens = total_tokens = None + + for line in raw_sse.splitlines(): + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + continue + try: + chunk = json.loads(data) + except json.JSONDecodeError: + continue + + # Usage chunk (sent when stream_options.include_usage=true): carries + # usage with an empty choices array. + if "usage" in chunk and chunk.get("choices") == []: + u = chunk["usage"] or {} + prompt_tokens = u.get("prompt_tokens") + completion_tokens = u.get("completion_tokens") + total_tokens = u.get("total_tokens") + continue + + for choice in chunk.get("choices", []): + delta_content = choice.get("delta", {}).get("content") + if delta_content: + content_parts.append(delta_content) + if choice.get("finish_reason"): + finish_reason = choice["finish_reason"] + + return ( + prompt_tokens, + completion_tokens, + total_tokens, + finish_reason, + "".join(content_parts) or None, + ) + + +def parse_responses_api_stream(raw_sse: str) -> _ParsedResponse: + """Same extraction for a Responses API SSE stream. + + The Responses API uses named events (``response.output_text.delta``, + ``response.completed``) rather than Chat Completions' ``choices[].delta`` + shape, so it needs its own parser — without this, every streaming Codex + call records null tokens and no response text. + """ + content_parts: list[str] = [] + finish_reason = None + prompt_tokens = completion_tokens = total_tokens = None + + for line in raw_sse.splitlines(): + if not line.startswith("data:"): + continue + data = line[5:].strip() + if not data or data == "[DONE]": + continue + try: + chunk = json.loads(data) + except json.JSONDecodeError: + continue + if not isinstance(chunk, dict): + continue + + event_type = chunk.get("type") or "" + if event_type.endswith("output_text.delta"): + delta = chunk.get("delta") + if isinstance(delta, str): + content_parts.append(delta) + elif event_type in ("response.completed", "response.incomplete"): + response = chunk.get("response") or {} + usage = response.get("usage") or {} + # The Responses API names these input/output rather than + # prompt/completion. + prompt_tokens = usage.get("input_tokens", usage.get("prompt_tokens")) + completion_tokens = usage.get( + "output_tokens", usage.get("completion_tokens") + ) + total_tokens = usage.get("total_tokens") + finish_reason = response.get("status") or ( + "completed" if event_type == "response.completed" else "incomplete" + ) + + return ( + prompt_tokens, + completion_tokens, + total_tokens, + finish_reason, + "".join(content_parts) or None, + ) + + +def extract_from_response(resp_json: dict) -> _ParsedResponse: + """Extract the same tuple from a non-streaming Chat Completions response.""" + usage = resp_json.get("usage", {}) or {} + choice = (resp_json.get("choices") or [{}])[0] + content = (choice.get("message", {}) or {}).get("content") or "" + return ( + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + choice.get("finish_reason"), + content or None, + ) + + +def extract_from_responses_api(resp_json: dict) -> _ParsedResponse: + """Extract the same tuple from a non-streaming Responses API response.""" + usage = resp_json.get("usage", {}) or {} + text_parts: list[str] = [] + for item in resp_json.get("output", []) or []: + if not isinstance(item, dict): + continue + for part in item.get("content", []) or []: + if isinstance(part, dict) and isinstance(part.get("text"), str): + text_parts.append(part["text"]) + return ( + usage.get("input_tokens", usage.get("prompt_tokens")), + usage.get("output_tokens", usage.get("completion_tokens")), + usage.get("total_tokens"), + resp_json.get("status"), + "".join(text_parts) or None, + ) diff --git a/src/agents/provider.py b/src/agents/provider.py new file mode 100644 index 0000000..3e99570 --- /dev/null +++ b/src/agents/provider.py @@ -0,0 +1,117 @@ +"""Upstream LLM provider resolution for the agent inference relay. + +Merge decision 6: rather than hardcoding a provider, every agent profile +configures a ``base_url`` + ``api_key_ref`` + ``model`` triple, and the upstream +only has to speak the OpenAI-compatible chat-completions wire format. That is a +superset of what either fork did on its own — a local Ollama install, Groq, +OpenRouter and OpenAI itself all work identically through this path, so +switching provider is a config change rather than a code change. + +Two consequences worth being explicit about: + +* API **keys are never stored in the database**. A profile stores + ``api_key_ref``, the *name* of the environment variable to read the key from + at request time, so dumping the profile table (or exposing it through the + admin UI) can't leak a credential. +* Codex's proprietary Responses API is not chat-completions-shaped, so it stays + a special-cased normalization path keyed on ``framework_version == "codex"`` + rather than being the default (see ``agents.normalize``). +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from typing import Optional + +# Fallback upstream used when a profile leaves `base_url` unset. Ollama's +# OpenAI-compatible endpoint is the default because it needs no credential and +# no account, so a fresh checkout works offline at zero cost. +DEFAULT_BASE_URL_ENV = "AGENT_UPSTREAM_BASE_URL" +DEFAULT_BASE_URL = "http://localhost:11434/v1" + +# Fallback env var consulted for the API key when a profile leaves +# `api_key_ref` unset. Local Ollama ignores the credential entirely, so an +# empty key is a valid, working configuration. +DEFAULT_API_KEY_ENV = "AGENT_UPSTREAM_API_KEY" + + +@dataclass(frozen=True) +class Upstream: + """A resolved OpenAI-compatible upstream target.""" + + base_url: str + api_key: str + model: str + # Which agent runtime this task belongs to: code4me2-agent | goose | codex. + framework_version: Optional[str] = None + + @property + def is_responses_api(self) -> bool: + """Whether this runtime speaks OpenAI's Responses API rather than + chat-completions. Only Codex does.""" + return (self.framework_version or "").strip().lower() == "codex" + + def endpoint(self, *, responses_api: bool) -> str: + """Full URL for the request, given the wire shape actually observed. + + The shape is taken from the request body (``input`` vs ``messages``) + rather than from ``framework_version`` alone, so a Codex profile driving + a chat-completions request still lands on the right path. + """ + suffix = "responses" if responses_api else "chat/completions" + return f"{self.base_url.rstrip('/')}/{suffix}" + + +def resolve_api_key(api_key_ref: Optional[str]) -> str: + """Read the upstream API key out of the environment. + + ``api_key_ref`` is an environment variable *name* taken from the agent + profile. An unset or empty value resolves to "" rather than raising: local + providers (Ollama, llama.cpp, vLLM without auth) don't need a credential, + and failing the request would make the zero-config path unusable. + """ + env_name = (api_key_ref or "").strip() or DEFAULT_API_KEY_ENV + key = os.getenv(env_name, "").strip() + if not key: + logging.info( + f"[Agent/provider] no API key in ${env_name} — forwarding unauthenticated " + f"(fine for a local provider, will 401 against a hosted one)" + ) + return key + + +def resolve_base_url(base_url: Optional[str]) -> str: + """Pick the upstream base URL: profile value, else env override, else Ollama.""" + candidate = (base_url or "").strip() + if candidate: + return candidate + return os.getenv(DEFAULT_BASE_URL_ENV, "").strip() or DEFAULT_BASE_URL + + +def resolve_upstream( + *, + model: str, + base_url: Optional[str] = None, + api_key_ref: Optional[str] = None, + framework_version: Optional[str] = None, +) -> Upstream: + """Build the upstream target for one inference call. + + Callers pass the values snapshotted onto the ``agent_task`` row at + task-creation time (not the live profile), so editing a profile mid-run + never changes where an in-flight task's calls go. + """ + resolved = Upstream( + base_url=resolve_base_url(base_url), + api_key=resolve_api_key(api_key_ref), + model=model, + framework_version=framework_version, + ) + logging.info( + f"[Agent/provider] upstream={resolved.base_url} model={resolved.model!r} " + f"runtime={resolved.framework_version or 'unknown'} " + f"authenticated={bool(resolved.api_key)}" + ) + return resolved diff --git a/src/agents/registry.py b/src/agents/registry.py new file mode 100644 index 0000000..23c4b6e --- /dev/null +++ b/src/agents/registry.py @@ -0,0 +1,63 @@ +"""Server-authoritative A/B assignment of agent profiles to users.""" + +from __future__ import annotations + +import logging +import random +import uuid +from typing import Optional + +from sqlalchemy.orm import Session + +from database import crud +from database.db_schemas import AgentProfile + + +def get_profile(db: Session, name: str) -> Optional[AgentProfile]: + """Resolve a profile by name. Returns None if it doesn't exist.""" + return crud.get_agent_profile(db, name) + + +def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[AgentProfile]: + """Return the agent profile assigned to ``user_id``. + + Assignment is server-authoritative and sticky: if the user already has an + assignment (auto or manual) it's returned unchanged, so a participant keeps + the same arm for the life of the experiment regardless of later changes to + the active set. Otherwise a profile is drawn at random (even split) and + persisted as an ``auto`` assignment. + + The client never picks its own arm — that's the point. A self-selected + bucket would invalidate the comparison. + + Candidate pool: when a study is active *and* has selected agent profiles, + the draw is restricted to that study's arms. Otherwise it falls back to all + active profiles, so behaviour is unchanged when no agent study is running + (backwards compatible with the completion-only platform). + + Returns ``None`` only when no candidate profiles exist at all, which the + caller surfaces as a 503 — minting a task with no profile would produce + telemetry that can't be attributed to any condition. + """ + assignment = crud.get_agent_profile_assignment(db, user_id) + if assignment is not None and assignment.profile is not None: + return assignment.profile + + candidates = crud.list_active_study_agent_profiles(db) + pool = "active study" + if not candidates: + candidates = crud.list_active_agent_profiles(db) + pool = "all active profiles" + if not candidates: + logging.error("[Agent/registry] no active agent profiles — cannot assign") + return None + + chosen = random.choice(candidates) + crud.set_agent_profile_assignment( + db, user_id=user_id, profile_id=chosen.profile_id, source="auto" + ) + logging.info( + f"[Agent/registry] drew arm {chosen.name!r} for user {str(user_id)[:8]}… " + f"from {pool} ({len(candidates)} candidate(s))" + ) + return chosen diff --git a/src/agents/telemetry.py b/src/agents/telemetry.py new file mode 100644 index 0000000..7ba98be --- /dev/null +++ b/src/agents/telemetry.py @@ -0,0 +1,129 @@ +"""Telemetry record captured per LLM inference call intercepted by the proxy. + +One ``InferenceRecord`` maps onto one ``agent_event`` row with +``event_type='model_call'``. Every field is a typed column — this is the +"columnar core" half of the merged schema; adapter-specific extras that don't +warrant a column go into ``agent_event.extra_json`` instead. + +Content fields (``first_system_message``, ``last_user_message``, +``response_text``) are only ever populated when the requesting user's +``store_agent_content`` preference resolves True, decided server-side. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +from pydantic import BaseModel, Field + + +class InferenceRecord(BaseModel): + request_id: str = Field( + description="UUID generated per call. Used as the model_call span_id." + ) + session_id: Optional[str] = Field( + default=None, + description="Code4Me session identifier, resolved server-side from the " + "session cookie. Links all inference calls made within one IDE session.", + ) + task_id: Optional[str] = Field( + default=None, + description="Agent task identifier. Links this call to an AgentTask row " + "so all inference steps of one task are grouped.", + ) + agent_profile: Optional[str] = Field( + default=None, + description="Profile name assigned for this task (e.g. 'default-goose'). " + "Allows per-arm analysis of inference behaviour.", + ) + event_type: str = Field( + default="model_call", + description="Event type for the agent_event table. Always 'model_call' " + "for proxy-captured inference calls.", + ) + timestamp: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc), + description="UTC timestamp when the request was received by the relay.", + ) + model: str = Field( + description="Model actually sent upstream, after the server-side override " + "from the task's snapshotted profile." + ) + streaming: bool = Field( + description="Whether the client requested a streaming response." + ) + message_count: int = Field( + description="Total messages in the conversation context sent to the model. " + "Indicates how deep into a task this call occurred." + ) + role_breakdown: dict[str, int] = Field( + default_factory=dict, + description="Count of messages per role (system, user, assistant, tool). " + "Captures the shape of the context window.", + ) + first_system_message: Optional[str] = Field( + default=None, + description="Text of the system-role message. Content — gated by " + "store_agent_content.", + ) + last_user_message: Optional[str] = Field( + default=None, + description="Text of the most recent user-role message. Content — gated by " + "store_agent_content.", + ) + tools_kept: int = Field( + default=0, + description="Number of tool definitions forwarded to the upstream model.", + ) + tools_stripped: int = Field( + default=0, + description="Number of tool definitions removed before forwarding, either " + "because the profile's allowlist excluded them or the provider rejects " + "their schema. A non-zero value here is how you discover a tool name " + "missing from agents.tools.KNOWN_AGENT_TOOLS.", + ) + tool_names_requested: list[str] = Field( + default_factory=list, + description="Function names of all tools in the client request before " + "filtering. Records what the agent had available regardless of which were " + "forwarded upstream.", + ) + max_tokens: Optional[int] = Field( + default=None, description="max_tokens parameter from the client request." + ) + active_file: Optional[str] = Field( + default=None, + description="Path of the file active in the IDE editor when this call was " + "made, supplied by the plugin via enrichment. Structural metadata — " + "stored regardless of the content-storage preference.", + ) + prompt_tokens: Optional[int] = Field( + default=None, + description="Input token count as reported by the upstream provider. None " + "for streaming calls when the provider returns no usage chunk.", + ) + completion_tokens: Optional[int] = Field( + default=None, description="Output token count as reported by the provider." + ) + total_tokens: Optional[int] = Field( + default=None, description="Total tokens as reported by the provider." + ) + finish_reason: Optional[str] = Field( + default=None, + description="Why the model stopped ('stop', 'tool_calls', 'length', or " + "'stream_aborted' when the relay lost the connection mid-flight). Useful " + "for detecting truncated responses.", + ) + response_text: Optional[str] = Field( + default=None, + description="Text of the model's response. Content — gated by " + "store_agent_content.", + ) + latency_ms: int = Field( + description="End-to-end relay latency in ms (request received → full " + "response returned). For streaming calls, until the last chunk is flushed." + ) + upstream_status: int = Field( + description="HTTP status code returned by the upstream provider." + ) diff --git a/src/agents/tools.py b/src/agents/tools.py new file mode 100644 index 0000000..d1ba1b1 --- /dev/null +++ b/src/agents/tools.py @@ -0,0 +1,88 @@ +"""Catalogue of tool names an agent profile can select. + +This backs ``GET /api/agent/available-tools``, which the admin UI uses to +populate the tool picker when creating a profile, and it is the fallback +allowlist the proxy filters against for tasks whose profile predates per-profile +tool selection. + +Two runtimes contribute names: + +* ``code4me2-agent`` — the built-in ReAct loop, whose tools are implemented in + ``code4me2_agent/file_tools.py`` and ``command_tools.py``. This list is + authoritative, because we own that code. +* ``goose`` — Block's Goose advertises its own extension tools over ACP. The + names below are the ones observed in practice; the set genuinely varies with + the installed Goose build and its enabled extensions, so it is a *best-known* + catalogue rather than a contract. Anything Goose sends that isn't listed + shows up in the ``tools_stripped`` count on the model_call event, which is how + you discover a name that needs adding. + +Codex contributes nothing here: it manages its own tool schemas over the +Responses API, and the proxy passes them through rather than filtering them +(see ``agents.normalize.normalize_responses_api_body``). +""" + +from __future__ import annotations + +# Built-in code4me2-agent tools. Each is allowlist-gated inside the runtime +# itself (paths confined to the workspace, commands checked against the +# profile's command allowlist). +CODE4ME2_AGENT_TOOLS: frozenset[str] = frozenset( + { + "read_file", + "write_file", + "create_file", + "replace_text", + "list_files", + "search_files", + "run_command", + } +) + +# Goose tool names observed over ACP. Grouped by the extension that provides +# them, since enabling/disabling a Goose extension moves a whole group. +GOOSE_TOOLS: frozenset[str] = frozenset( + { + # developer extension + "shell", + "read", + "write", + "tree", + "edit", + "analyze", + # extension manager + "extensionmanager__list_resources", + "extensionmanager__manage_extensions", + "extensionmanager__read_resource", + "extensionmanager__search_available_extensions", + # apps extension + "apps__create_app", + "apps__delete_app", + "apps__iterate_app", + "apps__list_apps", + # misc built-ins + "delegate", + "load", + "load_skill", + "todo__todo_write", + } +) + +# Union of everything a profile may legitimately name. +KNOWN_AGENT_TOOLS: frozenset[str] = CODE4ME2_AGENT_TOOLS | GOOSE_TOOLS + +# Per-runtime view, for a UI that wants to show only the relevant tools once a +# framework_version has been picked. +TOOLS_BY_FRAMEWORK: dict[str, frozenset[str]] = { + "code4me2-agent": CODE4ME2_AGENT_TOOLS, + "goose": GOOSE_TOOLS, + # Codex tools are self-managed and passed through untouched. + "codex": frozenset(), +} + + +def tools_for_framework(framework_version: str | None) -> frozenset[str]: + """Tools selectable for a runtime; the full catalogue when unrecognised.""" + if not framework_version: + return KNOWN_AGENT_TOOLS + return TOOLS_BY_FRAMEWORK.get(framework_version.strip().lower(), KNOWN_AGENT_TOOLS) diff --git a/src/backend/Responses.py b/src/backend/Responses.py index a7d024e..1999cda 100644 --- a/src/backend/Responses.py +++ b/src/backend/Responses.py @@ -1,4 +1,5 @@ from abc import ABC +from typing import Optional from uuid import UUID from fastapi.encoders import jsonable_encoder @@ -236,6 +237,97 @@ class ProjectNotFoundError(ErrorResponse): ) +# /api/acp — grant handoff for a locally launched agent process +class PrepareAcpGrantPostResponse(BaseResponse): + message: str = Field(default="ACP agent session prepared successfully.") + grant: str = Field(..., description="One-time ACP launch grant.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field(default=300, description="Grant lifetime.") + + +class ExchangeAcpGrantPostResponse(BaseResponse): + message: str = Field(default="ACP grant exchanged successfully.") + acp_token: str = Field(..., description="ACP process authorization token.") + project_id: str = Field(..., description="Authorized project id.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field(default=3600, description="ACP session lifetime.") + + +class ValidateAcpSessionPostResponse(BaseResponse): + message: str = Field(default="ACP authorization validated successfully.") + project_id: str = Field(..., description="Authorized project id.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field( + default=3600, description="Renewed ACP session lifetime." + ) + + +class AcpAgentConfigGetResponse(BaseResponse): + """Runtime configuration handed to a locally launched agent process. + + This is how the assigned A/B profile reaches the built-in runtime: rather + than the agent reading a local config file, the backend tells it which + model, provider endpoint, tools and step budget to use — so a study + assignment actually takes effect. + + ``api_key_ref`` is the *name* of an environment variable, never a key + value: the agent runs on the developer's machine and resolves the + credential from its own environment, so no secret transits this endpoint. + """ + + message: str = Field(default="ACP agent configuration retrieved successfully.") + agent_profile: Optional[str] = Field( + default=None, description="Assigned agent profile name." + ) + framework_version: Optional[str] = Field( + default=None, description="Agent runtime this profile targets." + ) + model: str = Field(description="Model the agent should use for completions.") + base_url: Optional[str] = Field( + default=None, + description="OpenAI-compatible provider base URL. Null = agent default.", + ) + api_key_ref: Optional[str] = Field( + default=None, + description="Name of the environment variable holding the provider API key.", + ) + commands_allowlist: list[str] = Field( + default_factory=list, + description="Terminal commands the agent may execute.", + ) + tools: list[str] = Field( + default_factory=list, description="Tool names the agent may call." + ) + max_iterations: int = Field( + default=8, description="Maximum tool-calling iterations.", ge=1 + ) + max_context_tokens: Optional[int] = Field( + default=None, description="Per-turn context window; null = model maximum." + ) + approval_policy: Optional[str] = Field( + default=None, description="How tool calls should be confirmed." + ) + temperature: Optional[float] = Field( + default=None, description="Sampling temperature; null = provider default." + ) + store_agent_content: bool = Field( + default=True, + description="Whether the server will persist content for this user. " + "Advisory only — the server enforces this regardless of what the agent " + "sends, so the agent may use it to avoid transmitting content " + "needlessly but cannot use it to enable capture.", + ) + + +class PersistentAuthTokenPostResponse(BaseResponse): + message: str = Field(default="Persistent auth token created successfully.") + auth_token: str = Field(..., description="Auth token without Redis expiration.") + + +class AcpAuthorizationError(ErrorResponse): + message: str = Field(default="ACP authorization is invalid or expired.") + + # /api/session/deactivate class DeactivateSessionPostResponse(BaseResponse): message: str = Field(default="Session deactivated successfully.") diff --git a/src/backend/acp_authorization.py b/src/backend/acp_authorization.py new file mode 100644 index 0000000..4c5bc19 --- /dev/null +++ b/src/backend/acp_authorization.py @@ -0,0 +1,320 @@ +"""Authorization for locally-launched ACP agent processes. + +The built-in ``code4me2-agent`` runtime runs as a separate OS process on the +developer's machine, started by the IDE plugin. It therefore has no session +cookie and cannot be authenticated the way the plugin is. This module implements +the handoff that gives it a credential of its own: + +1. **grant** — the authenticated plugin asks for a one-time launch grant, bound + to a specific user, session, project and workspace directory. The plugin + writes it where the agent process will find it. +2. **exchange** — the agent process trades the grant (single-use) for a longer + lived ``acp_session`` token. Consuming the grant means a leaked handoff file + can't be replayed after the agent has started. +3. **validate-or-refresh** — every subsequent agent request revalidates and + slides the session's expiry, so an idle agent expires but an active one + doesn't. + +Every scope is *derived*, never accepted from the caller. An agent request can +only ever act as the user, project and workspace that were bound into the grant +at step 1, and each revalidation re-checks that the parent plugin login and +session are still alive — so logging out in the IDE immediately invalidates the +agent process too, rather than leaving an orphaned credential valid for an hour. +""" + +from __future__ import annotations + +import secrets +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional, Protocol, Union + + +class TokenStore(Protocol): + """The subset of ``RedisManager`` this service needs. + + Declared as a Protocol so the authorization logic is testable against an + in-memory dict without a Redis instance. + """ + + def get( + self, type: str, token: str, reset_exp: bool = False + ) -> Optional[dict]: ... + + def set( + self, + type: str, + token: str, + info: dict, + force_reset_exp: bool = False, + ) -> None: ... + + def consume(self, type: str, token: str) -> Optional[dict]: ... + + def discard(self, type: str, token: str) -> None: ... + + +class AcpAuthorizationDenied(Exception): + """Raised for every authorization failure in this module. + + Deliberately carries a coarse message: the caller maps it to a flat 401 so + the response doesn't reveal *which* check failed. + """ + + +@dataclass(frozen=True) +class PreparedAcpGrant: + grant: str + workspace: str + expires_in_seconds: int = 300 + + +@dataclass(frozen=True) +class AcpSessionAuthorization: + acp_token: str + user_id: str + project_id: str + workspace: str + expires_in_seconds: int = 3600 + + +@dataclass(frozen=True) +class AcpServerAuthorization: + """A session scope plus the resolved project record. + + Needed by endpoints that act on project data (e.g. bridging a model turn + through the backend's own model registry), where the project's multi-file + context has to be loaded. + """ + + acp_token: str + user_id: str + session_id: str + project_id: str + project_info: dict[str, Any] + workspace: str + expires_in_seconds: int = 3600 + + +def authorize_acp_bearer( + token_store: TokenStore, + authorization: str, + *, + server_scope: bool = False, +) -> Optional[Union[AcpSessionAuthorization, AcpServerAuthorization]]: + """Resolve an ``Authorization: Bearer `` header to a scope. + + Returns None on any failure so route handlers can respond with a single + uniform 401 rather than branching on the reason. + """ + prefix = "Bearer " + if not authorization.startswith(prefix) or not authorization[len(prefix) :]: + return None + service = AcpAuthorizationService(token_store) + try: + if server_scope: + return service.resolve_server_scope(authorization[len(prefix) :]) + return service.validate_or_refresh(authorization[len(prefix) :]) + except AcpAuthorizationDenied: + return None + + +class AcpAuthorizationService: + def __init__( + self, + token_store: TokenStore, + *, + token_factory: Optional[Callable[[], str]] = None, + ) -> None: + self._tokens = token_store + self._token_factory = token_factory or (lambda: secrets.token_urlsafe(32)) + + def prepare_grant( + self, + *, + auth_token: str, + project_id: str, + workspace: str, + ) -> PreparedAcpGrant: + """Mint a one-time launch grant for an authenticated plugin. + + Any grant previously issued for this plugin session is discarded first, + so re-preparing (which the plugin does on every startup and login) + leaves exactly one usable grant rather than accumulating valid tokens. + """ + state = self._active_plugin_scope( + auth_token=auth_token, + project_id=project_id, + workspace=workspace, + ) + grant = self._token_factory() + pending = self._tokens.get("acp_pending_grant", state["parent_session_token"]) + if pending is not None and pending.get("grant"): + self._tokens.discard("acp_grant", pending["grant"]) + self._tokens.set("acp_grant", grant, state, force_reset_exp=True) + self._tokens.set( + "acp_pending_grant", + state["parent_session_token"], + {"grant": grant}, + force_reset_exp=True, + ) + return PreparedAcpGrant(grant=grant, workspace=state["workspace"]) + + def exchange_grant(self, grant: str) -> AcpSessionAuthorization: + """Trade a launch grant for an agent session token. + + The scope is re-checked *before* consuming, then the grant is consumed + atomically — so a grant whose parent session died in the meantime is + rejected, and a valid grant can only ever be redeemed once. + """ + state = self._tokens.get("acp_grant", grant) + if state is None: + raise AcpAuthorizationDenied( + "ACP grant is invalid or has already been used." + ) + self._require_active_scope(state) + consumed_state = self._tokens.consume("acp_grant", grant) + if consumed_state is None: + raise AcpAuthorizationDenied( + "ACP grant is invalid or has already been used." + ) + acp_token = self._token_factory() + self._tokens.set("acp_session", acp_token, consumed_state, force_reset_exp=True) + return AcpSessionAuthorization( + acp_token=acp_token, + user_id=consumed_state["user_id"], + project_id=consumed_state["project_id"], + workspace=consumed_state["workspace"], + ) + + def validate_or_refresh(self, acp_token: str) -> AcpSessionAuthorization: + state = self._validated_session_state(acp_token) + return AcpSessionAuthorization( + acp_token=acp_token, + user_id=state["user_id"], + project_id=state["project_id"], + workspace=state["workspace"], + ) + + def resolve_server_scope(self, acp_token: str) -> AcpServerAuthorization: + state = self._validated_session_state(acp_token) + project_info = self._tokens.get("project_token", state["project_id"]) + if project_info is None: + raise AcpAuthorizationDenied("An active project association is required.") + return AcpServerAuthorization( + acp_token=acp_token, + user_id=state["user_id"], + session_id=state["parent_session_token"], + project_id=state["project_id"], + project_info=project_info, + workspace=state["workspace"], + ) + + def _validated_session_state(self, acp_token: str) -> dict[str, str]: + """Revalidate an agent session and slide its expiry. + + A session whose parent plugin login has gone away is *discarded* here, + not merely rejected — otherwise it would sit in Redis until its TTL, + still resolvable if the user logged back in. + """ + state = self._tokens.get("acp_session", acp_token) + if state is None: + raise AcpAuthorizationDenied("ACP session is invalid or expired.") + try: + self._require_active_scope(state) + except AcpAuthorizationDenied: + self._tokens.discard("acp_session", acp_token) + raise + # Touch the parent session so an actively-working agent keeps the user's + # IDE session alive, matching how plugin requests behave. + self._tokens.get("session_token", state["parent_session_token"], reset_exp=True) + self._tokens.set("acp_session", acp_token, state, force_reset_exp=True) + return state + + def _active_plugin_scope( + self, + *, + auth_token: str, + project_id: str, + workspace: str, + ) -> dict[str, str]: + auth_info = self._tokens.get("auth_token", auth_token) + if auth_info is None or not auth_info.get("user_id"): + raise AcpAuthorizationDenied("An authenticated plugin login is required.") + user_id = auth_info["user_id"] + user_info = self._tokens.get("user_token", user_id) + if user_info is None or not user_info.get("session_token"): + raise AcpAuthorizationDenied("An active plugin session is required.") + return self._bound_scope( + auth_token=auth_token, + user_id=user_id, + session_token=user_info["session_token"], + project_id=project_id, + workspace=self._canonical_workspace(workspace), + ) + + def _require_active_scope(self, state: dict[str, str]) -> None: + """Re-check that everything the grant was bound to is still valid.""" + auth_info = self._tokens.get("auth_token", state["parent_auth_token"]) + if auth_info is None or auth_info.get("user_id") != state["user_id"]: + raise AcpAuthorizationDenied("The parent plugin login is no longer active.") + user_info = self._tokens.get("user_token", state["user_id"]) + if ( + user_info is None + or user_info.get("session_token") != state["parent_session_token"] + ): + raise AcpAuthorizationDenied( + "The parent plugin session is no longer active." + ) + self._bound_scope( + auth_token=state["parent_auth_token"], + user_id=state["user_id"], + session_token=state["parent_session_token"], + project_id=state["project_id"], + workspace=state["workspace"], + ) + + def _bound_scope( + self, + *, + auth_token: str, + user_id: str, + session_token: str, + project_id: str, + workspace: str, + ) -> dict[str, str]: + """Verify the session↔project association in both directions. + + Checking only one direction would let a stale association on either side + authorize an agent against a project the session no longer has open. + """ + session_info = self._tokens.get("session_token", session_token) + if session_info is None or project_id not in session_info.get( + "project_tokens", [] + ): + raise AcpAuthorizationDenied("An active project association is required.") + project_info = self._tokens.get("project_token", project_id) + if project_info is None or session_token not in project_info.get( + "session_tokens", [] + ): + raise AcpAuthorizationDenied("An active project association is required.") + return { + "user_id": user_id, + "parent_auth_token": auth_token, + "parent_session_token": session_token, + "project_id": project_id, + "workspace": workspace, + } + + @staticmethod + def _canonical_workspace(workspace: str) -> str: + """Resolve the workspace to a canonical absolute path. + + The agent's file tools confine themselves to this directory, so it must + be normalized before being bound into the grant — otherwise a relative + or ``..``-containing path could widen the agent's reach. + """ + path = Path(workspace).expanduser() + if not path.is_absolute(): + raise AcpAuthorizationDenied("A canonical absolute workspace is required.") + return path.resolve().as_posix() diff --git a/src/backend/redis_manager.py b/src/backend/redis_manager.py index ee9474d..dcd2d7d 100644 --- a/src/backend/redis_manager.py +++ b/src/backend/redis_manager.py @@ -62,6 +62,16 @@ def __get_exp(self, type: str) -> int: return self.auth_token_expires_in_seconds elif type == "session_token": return self.session_token_expires_in_seconds + elif type == "acp_grant": + # One-time launch grant handed to a locally-spawned agent process. + # Short-lived on purpose: it only has to survive the gap between the + # plugin writing the handoff file and the agent starting up. + return 300 + elif type == "acp_pending_grant": + return 300 + elif type == "acp_session": + # The agent process's working credential, refreshed on every use. + return 3600 elif type == "project_token": return -1 # project tokens do not expire by default elif type == "email_verification": @@ -129,6 +139,26 @@ def get(self, type: str, token: str, reset_exp: bool = False) -> Optional[dict]: return recursive_json_loads(data) # Parse JSON string to dict return None + def consume(self, type: str, token: str) -> Optional[dict]: + """Atomically retrieve and invalidate a one-time token. + + Used for ACP launch grants, where a replayable token would let anyone + who reads the handoff file mint an agent session. GETDEL makes the + exchange single-use even under concurrent attempts. + """ + if not token: + return None + data = self.__redis_client.getdel(f"{type}:{token}") + if data: + return recursive_json_loads(data) + return None + + def discard(self, type: str, token: str) -> None: + """Remove an ephemeral token without the DB-persistence side effects of + ``delete`` (which tears down whole session graphs).""" + if token: + self.__redis_client.delete(f"{type}:{token}") + def delete(self, type: str, token: str, db_session: Session): """ Delete token and related data from Redis and persist relevant info to DB. diff --git a/src/backend/routers/__init__.py b/src/backend/routers/__init__.py index 142ef62..18d49e4 100644 --- a/src/backend/routers/__init__.py +++ b/src/backend/routers/__init__.py @@ -1,6 +1,9 @@ from fastapi import APIRouter # Import sub-routers for different parts of the application +from .acp import router as acp_router +from .agent import router as agent_router +from .agents import router as agent_tasks_router from .chat import router as chat_router from .completion import router as completion_router from .project import router as project_router @@ -34,9 +37,20 @@ # Include WebSocket-based endpoints (e.g., completions, chat) router.include_router(ws_routers, prefix="/ws", tags=["WebSocket"]) -# Include analytics and monitoring endpoints +# Include analytics and monitoring endpoints router.include_router(analytics_router, prefix="/analytics", tags=["Analytics"]) +# Agent subsystem. Both routers share the /agent prefix but authenticate +# differently: `agent_router` covers profiles + A/B assignments (admin auth) and +# self-report ingestion + memory (ACP bearer auth from the locally launched +# agent process), while `agent_tasks_router` covers task lifecycle and the +# inference relay (plugin session cookie). +router.include_router(agent_router, prefix="/agent", tags=["Agent"]) +router.include_router(agent_tasks_router, prefix="/agent", tags=["Agent"]) + +# ACP grant handoff, so a locally launched agent process can authenticate. +router.include_router(acp_router, prefix="/acp", tags=["ACP"]) + @router.api_route("/ping", methods=["HEAD"]) def ping(): diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py new file mode 100644 index 0000000..95bb8f2 --- /dev/null +++ b/src/backend/routers/acp/__init__.py @@ -0,0 +1,360 @@ +"""ACP authorization and runtime configuration for locally launched agents. + +Endpoints (mounted under ``/api/acp``): + + POST /grant plugin mints a one-time launch grant + POST /session/exchange agent trades the grant for a session token + POST /session/validate-or-refresh agent revalidates and slides its expiry + GET /agent-config agent fetches its assigned runtime config + POST /persistent-auth-token admin mints a non-expiring auth token + +Why this exists: the built-in ``code4me2-agent`` runtime is a separate OS +process started by the IDE plugin. It has no session cookie, so it can't +authenticate the way the plugin does. The grant handoff gives it a credential of +its own, scoped to exactly one user, project and workspace directory. + +``/agent-config`` is where merge decision 6 lands for the built-in runtime: the +agent no longer reads a hardcoded local default, it asks the backend which +provider endpoint, model, tools and step budget to use. That's what makes the +assigned A/B profile actually govern the runtime we control, the same way the +inference relay governs the ones we don't. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Optional + +from fastapi import APIRouter, Cookie, Depends, Header + +import Queries +from App import App +from backend.acp_authorization import ( + AcpAuthorizationDenied, + AcpAuthorizationService, +) +from backend.Responses import ( + AcpAgentConfigGetResponse, + AcpAuthorizationError, + ExchangeAcpGrantPostResponse, + JsonResponseWithStatus, + PersistentAuthTokenPostResponse, + PrepareAcpGrantPostResponse, + ValidateAcpSessionPostResponse, +) +from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin +from database import crud +from utils import create_uuid + +router = APIRouter() + +# Fallback runtime config, used only when no agent profile can be resolved for +# the user (e.g. every profile has been deactivated). Deliberately minimal and +# read-only-ish: a handful of harmless inspection commands and no write tools, +# so an unconfigured agent degrades to something safe rather than to something +# powerful. +FALLBACK_COMMANDS_ALLOWLIST = ["pwd", "ls", "cat", "grep", "rg"] +FALLBACK_TOOLS = ["read_file", "list_files", "search_files"] +FALLBACK_MODEL = "qwen2.5-coder:7b" +FALLBACK_MAX_ITERATIONS = 6 + +_BEARER_PREFIX = "Bearer " + + +def _bearer_token(authorization: str) -> Optional[str]: + if not authorization.startswith(_BEARER_PREFIX): + return None + token = authorization[len(_BEARER_PREFIX) :] + return token or None + + +@router.post( + "/persistent-auth-token", + response_model=PersistentAuthTokenPostResponse, + status_code=201, +) +def create_persistent_auth_token( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +) -> JsonResponseWithStatus: + """Mint a non-expiring auth token for the calling admin. + + Needed for unattended runs (benchmark harnesses, long-lived dev setups) + where nobody is around to re-login when the normal token expires. Admin-only + and scoped to the caller's own user, since it deliberately sidesteps + session expiry. + """ + auth_token = create_uuid() + app.get_redis_manager().set( + "auth_token", + auth_token, + {"user_id": str(current_user.user_id)}, + ) + logging.info( + f"[ACP] persistent auth token minted for admin " + f"{str(current_user.user_id)[:8]}…" + ) + return JsonResponseWithStatus( + status_code=201, + content=PersistentAuthTokenPostResponse(auth_token=auth_token), + ) + + +@router.post("/grant", response_model=PrepareAcpGrantPostResponse, status_code=201) +def prepare_acp_grant( + request: Queries.PrepareAcpGrant, + app: App = Depends(App.get_instance), + auth_token: str = Cookie(""), +) -> JsonResponseWithStatus: + """Mint a one-time launch grant for the authenticated plugin. + + The plugin writes the returned grant where the agent process will find it. + Grants are short-lived and single-use, so a stale handoff file is inert. + """ + service = AcpAuthorizationService(app.get_redis_manager()) + try: + prepared = service.prepare_grant( + auth_token=auth_token, + project_id=str(request.project_id), + workspace=request.workspace, + ) + return JsonResponseWithStatus( + status_code=201, + content=PrepareAcpGrantPostResponse( + grant=prepared.grant, + workspace=prepared.workspace, + expires_in_seconds=prepared.expires_in_seconds, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +@router.post("/session/exchange", response_model=ExchangeAcpGrantPostResponse) +def exchange_acp_grant( + request: Queries.ExchangeAcpGrant, + app: App = Depends(App.get_instance), +) -> JsonResponseWithStatus: + """Trade a launch grant for the agent process's session token. + + Unauthenticated by design — possession of the grant *is* the credential. + That's safe because the grant is single-use, short-lived, and bound to a + scope the redeemer cannot influence. + """ + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.exchange_grant(request.grant) + return JsonResponseWithStatus( + status_code=200, + content=ExchangeAcpGrantPostResponse( + acp_token=session.acp_token, + project_id=session.project_id, + workspace=session.workspace, + expires_in_seconds=session.expires_in_seconds, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +@router.post( + "/session/validate-or-refresh", + response_model=ValidateAcpSessionPostResponse, +) +def validate_or_refresh_acp_session( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> JsonResponseWithStatus: + """Revalidate an agent session and slide its expiry. + + Also re-checks that the parent plugin login is still alive, so logging out + in the IDE invalidates the agent process rather than leaving it running with + a credential valid for another hour. + """ + token = _bearer_token(authorization) + if token is None: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.validate_or_refresh(token) + return JsonResponseWithStatus( + status_code=200, + content=ValidateAcpSessionPostResponse( + project_id=session.project_id, + workspace=session.workspace, + expires_in_seconds=session.expires_in_seconds, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +@router.get("/agent-config", response_model=AcpAgentConfigGetResponse) +def get_acp_agent_config( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> JsonResponseWithStatus: + """Return the runtime config for the calling agent process. + + Resolution order: + + 1. the user's assigned agent profile (the sticky A/B draw) — this is the + normal path, and it's what makes the assignment govern the runtime; + 2. the user's ``config`` row, whose optional ``agent`` section can override + the command allowlist (an operational safety setting rather than an + experimental condition, so it stays outside the profile); + 3. the conservative fallback constants above. + + No API key is ever returned — only ``api_key_ref``, the environment + variable name the agent reads it from locally. + """ + token = _bearer_token(authorization) + if token is None: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.validate_or_refresh(token) + except AcpAuthorizationDenied: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + + # Fallback defaults, overridden below where a profile/config supplies values. + commands_allowlist = list(FALLBACK_COMMANDS_ALLOWLIST) + tools = list(FALLBACK_TOOLS) + model = FALLBACK_MODEL + max_iterations = FALLBACK_MAX_ITERATIONS + agent_profile: Optional[str] = None + framework_version: Optional[str] = None + base_url: Optional[str] = None + api_key_ref: Optional[str] = None + approval_policy: Optional[str] = None + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + store_agent_content = False + + db = app.get_db_session() + try: + try: + user_uuid = uuid.UUID(str(session.user_id)) + except (ValueError, TypeError): + logging.warning( + f"[ACP/agent-config] scope user_id is not a UUID: {session.user_id!r}" + ) + user_uuid = None + + if user_uuid is not None: + # Import locally to avoid a circular import at module load time + # (agents.registry → database.crud → … → this router package). + from agents import registry + + profile = registry.resolve_assignment(db, user_uuid) + if profile is not None: + agent_profile = profile.name + framework_version = profile.framework_version + model = profile.model + base_url = profile.base_url + api_key_ref = profile.api_key_ref + approval_policy = profile.approval_policy + temperature = profile.temperature + max_context_tokens = profile.max_context_tokens + max_iterations = max(1, int(profile.max_steps or max_iterations)) + try: + parsed_tools = json.loads(profile.tools_json or "[]") + if isinstance(parsed_tools, list): + tools = [str(t) for t in parsed_tools] + except (json.JSONDecodeError, TypeError): + logging.warning( + f"[ACP/agent-config] profile {profile.name!r} has invalid " + f"tools_json — using fallback tool set" + ) + logging.info( + f"[ACP/agent-config] profile={profile.name!r} " + f"runtime={framework_version} model={model} " + f"tools={len(tools)} max_iterations={max_iterations}" + ) + else: + logging.warning( + "[ACP/agent-config] no active profile for user — " + "serving conservative fallback config" + ) + + store_agent_content = resolve_store_agent_content_for_acp( + db, session.user_id + ) + + # The command allowlist is an operational guardrail, so it can be + # narrowed per-user via the config row independently of the profile. + user = crud.get_user_by_id(db, user_uuid) if user_uuid else None + if user is not None: + config_row = crud.get_config_by_id(db, user.config_id) + if config_row is not None and config_row.config_data: + overrides = _parse_agent_config_section(config_row.config_data) + raw_allowlist = overrides.get("commands_allowlist") + if isinstance(raw_allowlist, list): + commands_allowlist = [ + str(c).strip() for c in raw_allowlist if c + ] + except Exception as error: + # Never fail the agent's startup on a config lookup problem — it would + # leave the developer with a dead agent rather than a degraded one. + logging.error( + f"[ACP/agent-config] config resolution failed, using fallbacks — {error}", + exc_info=True, + ) + finally: + db.close() + + return JsonResponseWithStatus( + status_code=200, + content=AcpAgentConfigGetResponse( + agent_profile=agent_profile, + framework_version=framework_version, + model=model, + base_url=base_url, + api_key_ref=api_key_ref, + commands_allowlist=commands_allowlist, + tools=tools, + max_iterations=max_iterations, + max_context_tokens=max_context_tokens, + approval_policy=approval_policy, + temperature=temperature, + store_agent_content=store_agent_content, + ), + ) + + +def _parse_agent_config_section(config_data: str) -> dict: + """Extract the optional ``agent`` section from a user's config blob. + + Configs are stored as JSON in this platform. Group-5's original version + also regex-parsed a HOCON ``agent { … }`` block; that is dropped here + because regex-parsing a config language is fragile and the JSON path is the + only shape this repo actually writes. A non-JSON config simply yields no + overrides rather than being half-parsed. + """ + try: + parsed = json.loads(config_data) + except (json.JSONDecodeError, ValueError, TypeError): + return {} + if not isinstance(parsed, dict): + return {} + agent_section = parsed.get("agent") + return agent_section if isinstance(agent_section, dict) else {} diff --git a/src/backend/routers/agent/__init__.py b/src/backend/routers/agent/__init__.py new file mode 100644 index 0000000..a160b92 --- /dev/null +++ b/src/backend/routers/agent/__init__.py @@ -0,0 +1,22 @@ +"""Agent sub-router: profiles, A/B assignments, telemetry ingestion, memory. + +Mounted at ``/api/agent`` alongside ``backend.routers.agents`` (task lifecycle +and the inference relay), which shares the same prefix. The split is by +*authentication*, not by URL space: + +* this package's ``ingest`` and ``memory`` routers authenticate the locally + launched agent process via its ACP bearer token; +* ``profiles`` is admin/authenticated-user via ``auth_utils``; +* ``backend.routers.agents`` authenticates the IDE plugin via its session cookie. +""" + +from fastapi import APIRouter + +from .ingest import router as ingest_router +from .memory import router as memory_router +from .profiles import router as profiles_router + +router = APIRouter() +router.include_router(profiles_router) +router.include_router(ingest_router) +router.include_router(memory_router) diff --git a/src/backend/routers/agent/acp_auth.py b/src/backend/routers/agent/acp_auth.py new file mode 100644 index 0000000..30b44d0 --- /dev/null +++ b/src/backend/routers/agent/acp_auth.py @@ -0,0 +1,66 @@ +"""Shared FastAPI dependency for ACP-bearer-authenticated agent endpoints. + +The built-in ``code4me2-agent`` runtime is a separate local process with no +session cookie, so it authenticates with the ``acp_session`` bearer token it got +from the grant exchange (see ``backend.acp_authorization``). This module turns +that header into a resolved scope for route handlers. + +Every failure mode collapses to one flat 401 so the response never reveals +*which* check failed. +""" + +from __future__ import annotations + +from typing import Optional + +from fastapi import Depends, Header, HTTPException + +from App import App +from backend.acp_authorization import ( + AcpServerAuthorization, + AcpSessionAuthorization, + authorize_acp_bearer, +) + +_UNAUTHORIZED_DETAIL = "ACP authorization is invalid or expired." + + +def require_acp_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> AcpSessionAuthorization: + """Resolve ``Authorization: Bearer `` to a session scope. + + The returned scope's ``user_id`` / ``project_id`` / ``workspace`` are + server-derived from the original grant, so handlers can treat them as + trusted and must never take those values from the request body instead. + """ + scope = authorize_acp_bearer(app.get_redis_manager(), authorization) + if not isinstance(scope, AcpSessionAuthorization): + raise HTTPException(status_code=401, detail=_UNAUTHORIZED_DETAIL) + return scope + + +def require_acp_server_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> AcpServerAuthorization: + """Same, but also resolves the project record. + + For handlers that need the project's stored context, not just its id. + """ + scope = authorize_acp_bearer( + app.get_redis_manager(), authorization, server_scope=True + ) + if not isinstance(scope, AcpServerAuthorization): + raise HTTPException(status_code=401, detail=_UNAUTHORIZED_DETAIL) + return scope + + +def optional_acp_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> Optional[AcpSessionAuthorization]: + """Resolve the scope if present, else None — for endpoints valid either way.""" + scope = authorize_acp_bearer(app.get_redis_manager(), authorization) + return scope if isinstance(scope, AcpSessionAuthorization) else None diff --git a/src/backend/routers/agent/consent.py b/src/backend/routers/agent/consent.py new file mode 100644 index 0000000..92ed5bd --- /dev/null +++ b/src/backend/routers/agent/consent.py @@ -0,0 +1,113 @@ +"""Server-side consent gate for persisting agent *content*. + +Merge decision 3: the mechanism is group-21's server-enforced double gate, with +the default flipped to ON. + +* **Server-enforced.** Whether message text, tool arguments/results, diffs and + raw payloads get written is decided here, from the user's stored preference — + never from a flag in the request body. A compromised or stale client cannot + turn content capture on, and cannot turn it off either (which matters for + research integrity as much as for privacy). +* **Default ON.** When a user row predates this preference the key is absent, + and it resolves to ``True``. This is a research platform where participants + give informed consent through a separate process before using it, so opt-out + is the right default for data collection — while the enforcement path stays + strict. + +Structural telemetry (token counts, latency, span tree, tool *names*, content +*lengths*) is never gated: it carries no user or code text, and it's what the +A/B analysis actually runs on. + +Every caller funnels through this module so there is exactly one place where the +decision is made. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Optional + +from sqlalchemy.orm import Session + +from database import crud +from database.db_schemas import ( + STORE_AGENT_CONTENT_DEFAULT, + STORE_AGENT_CONTENT_KEY, +) + + +def _preference_allows_content(preference: Optional[str]) -> bool: + """Read the flag out of a user's stored preference JSON. + + An absent key means the row predates the preference, so the documented + default applies. Malformed JSON is treated as absent rather than as a + refusal, because a corrupt preference blob shouldn't silently change what a + study collects — it should behave like an un-migrated row. + """ + if not preference: + return STORE_AGENT_CONTENT_DEFAULT + try: + parsed = json.loads(preference) + except (ValueError, TypeError): + logging.warning( + "[Agent/consent] unparseable user preference — " + f"falling back to default store_agent_content={STORE_AGENT_CONTENT_DEFAULT}" + ) + return STORE_AGENT_CONTENT_DEFAULT + if not isinstance(parsed, dict): + return STORE_AGENT_CONTENT_DEFAULT + return bool(parsed.get(STORE_AGENT_CONTENT_KEY, STORE_AGENT_CONTENT_DEFAULT)) + + +def resolve_store_agent_content_for_user( + db: Session, user_id: Optional[uuid.UUID] +) -> bool: + """Resolve the preference for a known user id. + + Returns False when the user can't be resolved at all: an unattributable + request has nobody's consent to rely on, so it gets the conservative answer + even though the default for a real user is True. + """ + if user_id is None: + return False + try: + user = crud.get_user_by_id(db, user_id) + except Exception as e: + logging.warning(f"[Agent/consent] user lookup failed for {user_id} — {e}") + return False + if user is None: + return False + return _preference_allows_content(user.preference) + + +def resolve_store_agent_content(db: Session, session_id: uuid.UUID) -> bool: + """Resolve the preference for the user owning ``session_id``. + + This is the plugin/proxy path, where the caller is identified by the + session cookie. + """ + try: + session = crud.get_session_by_id(db, session_id) + except Exception as e: + logging.warning(f"[Agent/consent] session lookup failed for {session_id} — {e}") + return False + if session is None or session.user_id is None: + return False + return resolve_store_agent_content_for_user(db, session.user_id) + + +def resolve_store_agent_content_for_acp(db: Session, raw_user_id: str) -> bool: + """Resolve the preference for an ACP-authorized agent process. + + ``raw_user_id`` comes from the server-derived ACP scope (see + ``backend.acp_authorization``), so it is trustworthy — but it arrives as a + string and has to parse as a UUID to be usable. + """ + try: + user_uuid = uuid.UUID(str(raw_user_id)) + except (ValueError, TypeError, AttributeError): + logging.warning(f"[Agent/consent] ACP scope user_id is not a UUID: {raw_user_id!r}") + return False + return resolve_store_agent_content_for_user(db, user_uuid) diff --git a/src/backend/routers/agent/ingest.py b/src/backend/routers/agent/ingest.py new file mode 100644 index 0000000..7ce1827 --- /dev/null +++ b/src/backend/routers/agent/ingest.py @@ -0,0 +1,319 @@ +"""Self-report telemetry ingestion for the built-in code4me2-agent runtime. + +Endpoints (mounted under ``/api/agent``): + + POST /events/ingest ingest a batch of self-reported events + GET /runs/{run_id} read back one run and its events + +This is the ingestion half of merge decision 2. Where third-party agents are +*observed* by the proxy relay, the runtime we own *reports* — and both write +into the same ``agent_event`` table, so no downstream consumer has to know which +runtime produced a row. + +The runtime mints its own ``run_id`` before it ever contacts the backend, so the +first batch for a run lazily creates the backing ``agent_task``, drawing the +user's assigned A/B profile at that point (the same sticky draw the plugin path +uses). Subsequent batches attach to the existing task. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from agents import ingest as ingest_module +from agents import lifecycle, registry +from App import App +from backend.acp_authorization import AcpSessionAuthorization +from backend.Responses import JsonResponseWithStatus +from backend.routers.agent.acp_auth import require_acp_scope +from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from database import crud + +router = APIRouter() + +# Terminal runtime statuses that should finalize the backing task. +_TERMINAL_RUN_STATUSES = frozenset({"completed", "failed", "cancelled", "error"}) + + +class AgentRunEnvelope(BaseModel): + """Run-level metadata accompanying every batch. + + ``owner_user_id`` / ``owner_project_id`` are deliberately absent: ownership + comes from the server-derived ACP scope, so a runtime can't claim someone + else's run. + """ + + run_id: str = Field(..., min_length=1) + session_id: str = Field(..., min_length=1) + status: str = Field(..., min_length=1) + source: str = Field(default="code4me2_agent", min_length=1) + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +class AgentEventEnvelope(BaseModel): + """One self-reported event, in the runtime's own envelope format. + + Kept loose on purpose: ``payload`` / ``metrics`` / ``observability`` are + free-form because the runtime evolves faster than this schema, and + ``agents.ingest`` decides which keys are promoted to typed columns versus + routed into ``extra_json``. Rejecting unknown keys here would mean a + runtime change silently dropping telemetry. + """ + + event_id: str = Field(..., min_length=1) + run_id: str = Field(..., min_length=1) + schema_version: str = Field(..., min_length=1) + event_type: str = Field(..., min_length=1) + timestamp: datetime + sequence: int = Field(..., ge=1) + source: str = Field(..., min_length=1) + session_id: str = Field(..., min_length=1) + request_id: str = Field(..., min_length=1) + parent_event_id: Optional[str] = None + message_id: Optional[str] = None + payload: dict[str, Any] = Field(default_factory=dict) + raw_payload: Optional[dict[str, Any]] = None + metrics: dict[str, Any] = Field(default_factory=dict) + # Descriptive only — the storage decision is made server-side. Retained for + # audit so a client/server mismatch is visible. + privacy: dict[str, Any] = Field(default_factory=dict) + observability: dict[str, Any] = Field(default_factory=dict) + + +class AgentEventBatchIngest(BaseModel): + run: AgentRunEnvelope + events: list[AgentEventEnvelope] = Field(default_factory=list) + + +def _resolve_or_create_task( + db, + *, + run: AgentRunEnvelope, + scope: AcpSessionAuthorization, +): + """Find the task backing this run, creating it on the first batch. + + Returns ``(task, created)``. Raises 403 if a task exists for this run id but + belongs to a different user — the run id is runtime-generated, so an + ownership check is the only thing preventing one agent process from writing + into another user's task. + """ + task = crud.get_agent_task_by_external_run_id(db, run.run_id) + + try: + owner_user_uuid = uuid.UUID(str(scope.user_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid user") + try: + owner_project_uuid = uuid.UUID(str(scope.project_id)) + except (ValueError, TypeError): + owner_project_uuid = None + + if task is not None: + if task.owner_user_id is not None and task.owner_user_id != owner_user_uuid: + logging.warning( + f"[Agent/ingest] 403 — run {run.run_id} owned by another user" + ) + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + return task, False + + profile = registry.resolve_assignment(db, owner_user_uuid) + if profile is None: + raise HTTPException( + status_code=503, + detail="No active agent profiles are configured on the server", + ) + + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + source="code4me2_agent", + owner_user_id=owner_user_uuid, + owner_project_id=owner_project_uuid, + external_run_id=run.run_id, + agent_session_id=run.session_id, + status="running", + started_at=run.started_at, + ) + logging.info( + f"[Agent/ingest] created task {task.task_id} for run {run.run_id} " + f"profile={profile.name!r} user={str(owner_user_uuid)[:8]}…" + ) + return task, True + + +@router.post("/events/ingest", summary="Ingest self-reported agent telemetry") +def ingest_agent_events( + body: AgentEventBatchIngest, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Ingest a batch of events reported by the built-in runtime. + + Idempotent by the runtime's own event ids: a retried batch reports its + duplicates rather than double-writing, because the uploader can't tell + whether a timed-out POST was applied. + """ + if any(event.run_id != body.run.run_id for event in body.events): + raise HTTPException( + status_code=400, detail="All events must belong to the request run_id." + ) + + db = app.get_db_session() + try: + task, _created = _resolve_or_create_task(db, run=body.run, scope=scope) + + # Consent is resolved from the *server's* record of the user's + # preference, not from the `privacy` block in the payload. + content_included = resolve_store_agent_content_for_acp(db, scope.user_id) + + ingested, skipped = ingest_module.ingest_event_batch( + db, + task_id=task.task_id, + events=[e.model_dump() for e in body.events], + content_included=content_included, + agent_profile=task.agent_profile, + ) + + # A run reporting a terminal status is finished, so aggregate it now + # rather than waiting for a close call the runtime may never make. + if body.run.status.strip().lower() in _TERMINAL_RUN_STATUSES: + status = ( + "failed" + if body.run.status.strip().lower() in ("failed", "error") + else "done" + ) + lifecycle.finalize_agent_task(db, task.task_id, status=status) + + return JsonResponseWithStatus( + status_code=201, + content={ + "run_id": body.run.run_id, + "task_id": str(task.task_id), + "ingested_event_count": ingested, + "duplicate_event_ids": skipped, + "content_stored": content_included, + }, + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + logging.error(f"[Agent/ingest] error ingesting events: {error}", exc_info=True) + raise HTTPException( + status_code=500, detail="Server failed to ingest agent events." + ) + finally: + db.close() + + +@router.get("/runs/{run_id}", summary="Read back a self-reported agent run") +def get_agent_run( + run_id: str, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Return one run's task row and its events, for the owning agent process.""" + db = app.get_db_session() + try: + task = crud.get_agent_task_by_external_run_id(db, run_id) + if task is None: + raise HTTPException(status_code=404, detail="Agent run not found.") + try: + owner_user_uuid = uuid.UUID(str(scope.user_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid user") + if task.owner_user_id != owner_user_uuid: + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + + events = crud.get_agent_events_by_task(db, task.task_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "run": _serialize_task(task), + "events": [_serialize_event(e) for e in events], + }, + ) + except HTTPException: + raise + except Exception as error: + logging.error( + f"[Agent/ingest] error retrieving run {run_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to retrieve agent run." + ) + finally: + db.close() + + +def _iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + +def _serialize_task(task) -> dict[str, Any]: + return { + "task_id": str(task.task_id), + "run_id": task.external_run_id, + "agent_session_id": task.agent_session_id, + "status": task.status, + "source": task.source, + "agent_profile": task.agent_profile, + "model": task.model, + "framework_version": task.framework_version, + "total_steps": task.total_steps, + "input_tokens": task.input_tokens, + "output_tokens": task.output_tokens, + "started_at": _iso(task.started_at), + "completed_at": _iso(task.completed_at), + "created_at": _iso(task.created_at), + } + + +def _serialize_event(event) -> dict[str, Any]: + """Serialise an event row. + + Content columns are echoed as stored — which means null when consent was + off. Nothing is reconstructed here, so this endpoint can't be used to read + back content the gate refused to persist. + """ + return { + "event_id": str(event.event_id), + "event_index": event.event_index, + "event_type": event.event_type, + "source": event.source, + "schema_version": event.schema_version, + "span_id": event.span_id, + "parent_span_id": str(event.parent_span_id) if event.parent_span_id else None, + "request_id": event.request_id, + "model": event.model, + "latency_ms": event.latency_ms, + "prompt_tokens": event.prompt_tokens, + "completion_tokens": event.completion_tokens, + "total_tokens": event.total_tokens, + "finish_reason": event.finish_reason, + "tool_name": event.tool_name, + "tool_arguments_length": event.tool_arguments_length, + "tool_result_length": event.tool_result_length, + "extra_json": event.extra_json, + "occurred_at": _iso(event.occurred_at), + "created_at": _iso(event.created_at), + } diff --git a/src/backend/routers/agent/memory.py b/src/backend/routers/agent/memory.py new file mode 100644 index 0000000..9aa7164 --- /dev/null +++ b/src/backend/routers/agent/memory.py @@ -0,0 +1,197 @@ +"""Persistent per-session agent memory. + +Endpoints (mounted under ``/api/agent``): + + GET /memory/{session_id} read the stored memory snapshot + PUT /memory/{session_id} replace the stored memory snapshot + DELETE /memory/{session_id} forget a session's memory + +This is the capability group-5 had and group-21 didn't: agent conversation state +survives an IDE (or agent process) restart, so a developer who closes their +editor mid-task can pick the conversation back up instead of starting over. + +Ownership is server-derived from the ACP scope on every call, so a runtime can +only read or write memory for sessions belonging to its own user and project — +the ``session_id`` in the path is a lookup key, not an authorization claim. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.acp_authorization import AcpSessionAuthorization +from backend.Responses import JsonResponseWithStatus +from backend.routers.agent.acp_auth import require_acp_scope +from database import crud + +router = APIRouter() + + +class AgentMemorySnapshot(BaseModel): + """The runtime's serialized memory window. + + Messages are stored opaquely: the shape is the runtime's business, and + schema-checking it here would couple the backend to the adapter's internal + context format. + """ + + messages: list[dict[str, Any]] = Field(default_factory=list) + + +def _scope_owns(scope: AcpSessionAuthorization, memory) -> bool: + return ( + memory.owner_user_id == str(scope.user_id) + and memory.owner_project_id == str(scope.project_id) + ) + + +def _serialize(memory) -> dict[str, Any]: + try: + payload = json.loads(memory.memory_json) or {} + except (ValueError, TypeError): + logging.warning( + f"[Agent/memory] unparseable snapshot for session {memory.session_id}" + ) + payload = {} + messages = payload.get("messages", []) + if not isinstance(messages, list): + messages = [] + return { + "session_id": memory.session_id, + "messages": messages, + "updated_at": _iso(memory.updated_at), + "created_at": _iso(memory.created_at), + } + + +def _iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + +@router.get("/memory/{session_id}", summary="Read persisted agent session memory") +def get_agent_memory( + session_id: str, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + db = app.get_db_session() + try: + memory = crud.get_agent_memory_by_session_id(db, session_id) + if memory is None: + # A fresh session legitimately has no memory yet — the runtime + # treats 404 as "start empty", so this is not an error path. + raise HTTPException(status_code=404, detail="Agent memory not found.") + if not _scope_owns(scope, memory): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + return JsonResponseWithStatus(status_code=200, content=_serialize(memory)) + except HTTPException: + raise + except Exception as error: + logging.error( + f"[Agent/memory] error reading {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to retrieve agent memory." + ) + finally: + db.close() + + +@router.put("/memory/{session_id}", summary="Replace persisted agent session memory") +def upsert_agent_memory( + session_id: str, + body: AgentMemorySnapshot, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Replace the snapshot for one session. + + Note this is *not* gated by ``store_agent_content``. Memory is operational + state the agent needs to keep working across restarts — it's the + conversation itself, not telemetry about it, and gating it would silently + break the feature rather than protect anything. It is scoped to its owner, + replaced wholesale on every write, and deletable via DELETE. + """ + db = app.get_db_session() + try: + existing = crud.get_agent_memory_by_session_id(db, session_id) + if existing is not None and not _scope_owns(scope, existing): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + memory = crud.upsert_agent_memory( + db, + session_id=session_id, + owner_user_id=str(scope.user_id), + owner_project_id=str(scope.project_id), + messages=body.messages, + ) + return JsonResponseWithStatus( + status_code=200, + content={ + **_serialize(memory), + "message_count": len(body.messages), + }, + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + logging.error( + f"[Agent/memory] error saving {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to save agent memory." + ) + finally: + db.close() + + +@router.delete("/memory/{session_id}", summary="Forget a session's agent memory") +def delete_agent_memory( + session_id: str, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Delete a session's memory. + + Gives the runtime (and through it the user) a way to actually discard + conversation state, rather than only overwriting it. + """ + db = app.get_db_session() + try: + existing = crud.get_agent_memory_by_session_id(db, session_id) + if existing is None: + raise HTTPException(status_code=404, detail="Agent memory not found.") + if not _scope_owns(scope, existing): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + crud.delete_agent_memory(db, session_id) + return JsonResponseWithStatus( + status_code=200, content={"deleted": True, "session_id": session_id} + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + logging.error( + f"[Agent/memory] error deleting {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to delete agent memory." + ) + finally: + db.close() diff --git a/src/backend/routers/agent/profiles.py b/src/backend/routers/agent/profiles.py new file mode 100644 index 0000000..6714b4a --- /dev/null +++ b/src/backend/routers/agent/profiles.py @@ -0,0 +1,418 @@ +"""Admin CRUD for agent profiles, plus the plugin-facing registry. + +Endpoints (mounted under ``/api/agent``): + + GET /available-tools tool names a profile may select + GET /profiles list profiles (admin) + POST /profiles create a profile (admin) + PUT /profiles/{id} update a profile (admin) + DELETE /profiles/{id} delete a profile (admin) + GET /registry profiles in the shape the plugin consumes + GET /assignments list A/B assignments (admin) + PUT /assignments/{user_id} pin a user to a profile (admin) + DELETE /assignments/{user_id} clear an assignment, re-rolling on next task + +Profiles are the unit of experimental control: a profile fixes the runtime, the +provider, the model, the tool allowlist and the approval policy, and users are +assigned to one as an A/B arm. +""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field, field_validator +from sqlalchemy.exc import IntegrityError + +from agents.tools import KNOWN_AGENT_TOOLS, tools_for_framework +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from database import crud +from database.db_schemas import AgentProfile, AgentProfileAssignment + +router = APIRouter() + +# Runtimes a profile may target. Anything else would mint tasks the plugin has +# no way to launch, so it's rejected at the API boundary rather than failing +# later at agent-startup time. +SUPPORTED_FRAMEWORKS = ("code4me2-agent", "goose", "codex") + + +class AgentProfilePayload(BaseModel): + name: str = Field(..., min_length=1) + model: str = Field(..., min_length=1) + framework_version: str = Field(default="code4me2-agent") + # Generic OpenAI-compatible endpoint. None = fall back to the server default. + base_url: Optional[str] = Field(default=None) + # Name of the env var holding the upstream key — never the key itself, so a + # profile is always safe to read back over the API. + api_key_ref: Optional[str] = Field(default=None) + tools_json: str = Field(default="[]") + approval_policy: str = Field(..., min_length=1) + max_steps: int = Field(..., ge=1) + is_active: bool = Field(default=True) + # None = don't inject; let the provider use its own default. + temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) + max_context_tokens: Optional[int] = Field(default=None, ge=1) + + @field_validator("tools_json") + @classmethod + def validate_tools_json(cls, value: str) -> str: + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("tools_json must be a valid JSON array") from exc + if not isinstance(parsed, list) or not all( + isinstance(item, str) for item in parsed + ): + raise ValueError("tools_json must be a JSON array of strings") + # Re-serialise so stored values are canonical regardless of input spacing. + return json.dumps(parsed) + + @field_validator("framework_version") + @classmethod + def validate_framework(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in SUPPORTED_FRAMEWORKS: + raise ValueError( + f"framework_version must be one of {', '.join(SUPPORTED_FRAMEWORKS)}" + ) + return normalized + + @field_validator("api_key_ref") + @classmethod + def validate_api_key_ref(cls, value: Optional[str]) -> Optional[str]: + """Reject anything that looks like a key rather than a variable name. + + This is a guardrail against the obvious mistake: pasting the actual + secret into the field. Env var names are short and uppercase, so a long + value or one containing key-ish punctuation is almost certainly a + credential that must not be persisted. + """ + if value is None: + return None + candidate = value.strip() + if not candidate: + return None + if len(candidate) > 128 or any(ch in candidate for ch in "-. /:"): + raise ValueError( + "api_key_ref must be the NAME of an environment variable " + "(e.g. OPENAI_API_KEY), not an API key value" + ) + return candidate + + +def _profile_to_dict(profile: AgentProfile) -> dict[str, Any]: + return { + "profile_id": str(profile.profile_id), + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "base_url": profile.base_url, + "api_key_ref": profile.api_key_ref, + "tools_json": profile.tools_json, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "is_active": profile.is_active, + "temperature": profile.temperature, + "max_context_tokens": profile.max_context_tokens, + "created_at": ( + profile.created_at.isoformat() + if isinstance(profile.created_at, datetime) + else profile.created_at + ), + } + + +def _profile_to_registry_dict(profile: AgentProfile) -> dict[str, Any]: + """Shape a profile for the plugin, including the env the runtime expects. + + Note that ``api_key_ref`` is exposed but the key is not: the plugin only + ever launches a *local* agent process, and the process reads the credential + from the backend at runtime, so the secret never transits this endpoint. + """ + try: + tools = json.loads(profile.tools_json or "[]") + except json.JSONDecodeError: + tools = [] + + env = { + "SECURITY_CONFIRMATION_MODE": profile.approval_policy, + "MAX_ITERATIONS": str(profile.max_steps), + "CODE4ME_AGENT_PROFILE": profile.name, + } + if not tools: + env["DISABLE_TOOLS"] = "true" + + return { + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "base_url": profile.base_url, + "api_key_ref": profile.api_key_ref, + "tools": tools, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "max_context_tokens": profile.max_context_tokens, + "is_active": profile.is_active, + "temperature": profile.temperature, + "env": env, + } + + +@router.get("/available-tools", summary="List tool names a profile may select") +def list_available_tools( + framework_version: Optional[str] = Query( + default=None, + description="Restrict to the tools relevant for one runtime " + "(code4me2-agent | goose | codex). Omit for the full catalogue.", + ), + current_user: AuthenticatedUser = Depends(require_admin), +): + tools = ( + tools_for_framework(framework_version) + if framework_version + else KNOWN_AGENT_TOOLS + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "tools": sorted(tools), + "framework_version": framework_version, + "frameworks": list(SUPPORTED_FRAMEWORKS), + }, + ) + + +@router.get("/profiles", summary="List agent profiles") +def list_agent_profiles( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + profiles = crud.list_agent_profiles(db) + return JsonResponseWithStatus( + status_code=200, + content={"profiles": [_profile_to_dict(p) for p in profiles]}, + ) + finally: + db.close() + + +@router.post("/profiles", summary="Create an agent profile") +def create_agent_profile( + payload: AgentProfilePayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + profile = crud.create_agent_profile( + db, + name=payload.name, + model=payload.model, + framework_version=payload.framework_version, + base_url=payload.base_url, + api_key_ref=payload.api_key_ref, + tools_json=payload.tools_json, + approval_policy=payload.approval_policy, + max_steps=payload.max_steps, + is_active=payload.is_active, + temperature=payload.temperature, + max_context_tokens=payload.max_context_tokens, + ) + return JsonResponseWithStatus( + status_code=201, content={"profile": _profile_to_dict(profile)} + ) + except IntegrityError as exc: + db.rollback() + raise HTTPException( + status_code=409, detail="Agent profile name already exists" + ) from exc + finally: + db.close() + + +@router.put("/profiles/{profile_id}", summary="Update an agent profile") +def update_agent_profile( + profile_id: uuid.UUID, + payload: AgentProfilePayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + """Update a profile. + + Note this does *not* retroactively change tasks already running: the + experimental conditions (model, temperature, policy, tools) were snapshotted + onto each ``agent_task`` row at creation. The provider triple is the + exception and is read live, so rotating a credential or endpoint takes + effect on the next call. + """ + db = app.get_db_session() + try: + profile = crud.update_agent_profile( + db, + profile_id=profile_id, + name=payload.name, + model=payload.model, + framework_version=payload.framework_version, + base_url=payload.base_url, + api_key_ref=payload.api_key_ref, + tools_json=payload.tools_json, + approval_policy=payload.approval_policy, + max_steps=payload.max_steps, + is_active=payload.is_active, + temperature=payload.temperature, + max_context_tokens=payload.max_context_tokens, + ) + if profile is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + return JsonResponseWithStatus( + status_code=200, content={"profile": _profile_to_dict(profile)} + ) + except IntegrityError as exc: + db.rollback() + raise HTTPException( + status_code=409, detail="Agent profile name already exists" + ) from exc + finally: + db.close() + + +@router.delete("/profiles/{profile_id}", summary="Delete an agent profile") +def delete_agent_profile( + profile_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + """Delete a profile. + + Deleting cascades to its assignments but leaves historical ``agent_task`` + rows intact (they reference the profile by name, not id), so past telemetry + stays attributable. To retire an arm mid-study prefer setting + ``is_active=false``, which stops new draws without disturbing assigned users. + """ + db = app.get_db_session() + try: + deleted = crud.delete_agent_profile(db, profile_id) + if not deleted: + raise HTTPException(status_code=404, detail="Agent profile not found") + return JsonResponseWithStatus( + status_code=200, + content={"deleted": True, "profile_id": str(profile_id)}, + ) + finally: + db.close() + + +@router.get("/registry", summary="List profiles available to the plugin") +def list_agent_registry( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + profiles = crud.list_agent_profiles(db) + return JsonResponseWithStatus( + status_code=200, + content={"profiles": [_profile_to_registry_dict(p) for p in profiles]}, + ) + finally: + db.close() + + +# ── A/B assignments ───────────────────────────────────────────────────────── +# +# Assignments are normally created automatically (sticky random draw) on a +# user's first agent task. These admin endpoints let you inspect them and pin a +# specific user to a specific profile (source="manual"), which overrides the +# random draw and is excluded from A/B analysis. + + +class AssignmentPayload(BaseModel): + profile_id: uuid.UUID + + +def _assignment_to_dict(assignment: AgentProfileAssignment) -> dict[str, Any]: + return { + "user_id": str(assignment.user_id), + "profile_id": str(assignment.profile_id), + "profile_name": assignment.profile.name if assignment.profile else None, + "source": assignment.source, + "assigned_at": ( + assignment.assigned_at.isoformat() + if isinstance(assignment.assigned_at, datetime) + else assignment.assigned_at + ), + } + + +@router.get("/assignments", summary="List all agent profile assignments") +def list_agent_assignments( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + assignments = crud.list_agent_profile_assignments(db) + return JsonResponseWithStatus( + status_code=200, + content={"assignments": [_assignment_to_dict(a) for a in assignments]}, + ) + finally: + db.close() + + +@router.put( + "/assignments/{user_id}", summary="Manually pin a user to an agent profile" +) +def set_agent_assignment( + user_id: uuid.UUID, + payload: AssignmentPayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + if crud.get_agent_profile_by_id(db, payload.profile_id) is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + assignment = crud.set_agent_profile_assignment( + db, user_id=user_id, profile_id=payload.profile_id, source="manual" + ) + return JsonResponseWithStatus( + status_code=200, content={"assignment": _assignment_to_dict(assignment)} + ) + finally: + db.close() + + +@router.delete( + "/assignments/{user_id}", + summary="Clear a user's assignment (re-rolls on next task)", +) +def delete_agent_assignment( + user_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + deleted = crud.delete_agent_profile_assignment(db, user_id) + if not deleted: + raise HTTPException(status_code=404, detail="Assignment not found") + return JsonResponseWithStatus( + status_code=200, + content={"deleted": True, "user_id": str(user_id)}, + ) + finally: + db.close() diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py new file mode 100644 index 0000000..1413971 --- /dev/null +++ b/src/backend/routers/agents.py @@ -0,0 +1,471 @@ +"""Agent task lifecycle and the inference relay, for the plugin-facing paths. + +Endpoints (mounted under ``/api/agent``): + + POST /task mint a task, assigning the A/B arm server-side + POST /task/{id}/close finalize a task and aggregate its telemetry + POST /task/{id}/telemetry bulk-upload OTel spans for one finished session + POST /inference the authed relay third-party agents talk through + +All four authenticate via the plugin's ``session_token`` cookie, and every task +operation re-checks that the task belongs to the calling session — so one +developer's plugin can't read or write another's telemetry. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any, Optional + +from fastapi import APIRouter, Cookie, Depends, HTTPException, Response +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field + +from agents import inference, lifecycle, registry +from App import App +from backend.routers.agent.consent import resolve_store_agent_content +from database import crud + +router = APIRouter(tags=["Agent"]) + + +def require_session( + app: App = Depends(App.get_instance), + session_token: str = Cookie(""), +) -> uuid.UUID: + """Validate the session_token cookie via Redis and return the session_id UUID. + + The cookie value *is* the session_id primary key in the DB session table. + Raises 401 if the cookie is missing, expired, or malformed. + """ + redis_manager = app.get_redis_manager() + session_info = redis_manager.get("session_token", session_token) + if session_info is None or not session_info.get("user_token"): + raise HTTPException(status_code=401, detail="Invalid or expired session token") + try: + return uuid.UUID(session_token) + except ValueError: + raise HTTPException(status_code=401, detail="Malformed session token") + + +def _require_own_task(db, task_id: uuid.UUID, session_id: uuid.UUID): + """Load a task, enforcing that it belongs to the calling session.""" + task = crud.get_agent_task(db, task_id) + if task is None: + raise HTTPException(status_code=404, detail="Agent task not found") + if task.session_id is None or task.session_id != session_id: + logging.warning( + f"[Agent] 403 — task {task_id} session mismatch " + f"(task.session={str(task.session_id)[:8]}… vs " + f"cookie={str(session_id)[:8]}…)" + ) + raise HTTPException( + status_code=403, detail="Task does not belong to this session" + ) + return task + + +# ── Task creation ──────────────────────────────────────────────────────────── + + +class TaskCreateRequest(BaseModel): + task_id: Optional[uuid.UUID] = None + task_description: Optional[str] = None + # NOTE: the client does not choose its profile. The arm is assigned + # server-side (see registry.resolve_assignment) so A/B buckets can't be + # self-selected. Any `profile` field sent by an older client is ignored. + + +@router.post("/task") +def create_agent_task( + body: TaskCreateRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> JSONResponse: + """Mint an agent task with the assigned profile snapshotted onto it. + + Accepting a client-supplied ``task_id`` makes this idempotent: the plugin + generates the id before launching the agent, and a retry after a flaky + response returns 200 with the same task rather than creating a duplicate. + A task id already claimed by a *different* session is a 409, not a silent + takeover. + """ + logging.info( + f"[Agent/task] request — task_id={body.task_id} " + f"session={str(session_id)[:8]}…" + ) + + db = app.get_db_session() + try: + if body.task_id is not None: + existing = crud.get_agent_task(db, body.task_id) + if existing is not None: + if existing.session_id != session_id: + logging.warning( + f"[Agent/task] 409 — task {body.task_id} belongs to another " + f"session" + ) + raise HTTPException( + status_code=409, + detail="Task ID already exists under a different session " + "— use a new task_id", + ) + logging.info( + f"[Agent/task] idempotent — task {body.task_id} already owned " + f"by this session" + ) + return JSONResponse( + {"task_id": str(existing.task_id)}, status_code=200 + ) + + session = crud.get_session_by_id(db, session_id) + if session is None or session.user_id is None: + raise HTTPException( + status_code=401, detail="Session is not associated with a user" + ) + + profile = registry.resolve_assignment(db, session.user_id) + if profile is None: + raise HTTPException( + status_code=503, + detail="No active agent profiles are configured on the server", + ) + + # task_description is the user's own words — content, so honour the + # consent gate even at creation time. + content_included = resolve_store_agent_content(db, session_id) + + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + task_description=body.task_description if content_included else None, + session_id=session_id, + owner_user_id=session.user_id, + task_id=body.task_id, + source="plugin", + ) + logging.info( + f"[Agent/task] created task_id={task.task_id} profile={profile.name!r} " + f"runtime={profile.framework_version} model={profile.model} " + f"user={str(session.user_id)[:8]}…" + ) + return JSONResponse( + { + "task_id": str(task.task_id), + # The plugin needs the runtime name to know which agent to launch. + "framework_version": profile.framework_version, + "agent_profile": profile.name, + }, + status_code=201, + ) + except HTTPException: + raise + except Exception as e: + logging.error(f"[Agent/task] DB error creating task — {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Failed to create task") + finally: + db.close() + + +# ── Inference relay ────────────────────────────────────────────────────────── + + +class InferenceRequest(BaseModel): + task_id: uuid.UUID + request: dict[str, Any] + enrichment: Optional[dict[str, Any]] = None + + +@router.post("/inference") +async def run_agent_inference( + body: InferenceRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Authed relay for third-party agents (Goose, Codex). + + The agent talks to the plugin's local proxy, which forwards here with the + session cookie attached. Upstream target, model, temperature and tool policy + all come from the profile snapshotted onto the task — not from what the + agent asked for — so the assigned A/B arm actually takes effect. + """ + logging.info( + f"[Agent/inference] request — task_id={body.task_id} " + f"session={str(session_id)[:8]}… " + f"model={body.request.get('model', '?')} " + f"stream={body.request.get('stream', False)} " + f"messages={len(body.request.get('messages', []))}" + ) + + db = app.get_db_session() + try: + task = _require_own_task(db, body.task_id, session_id) + # Resolved server-side from the stored preference, never from the + # request body — see backend.routers.agent.consent. + content_included = resolve_store_agent_content(db, session_id) + # The profile is looked up only for the provider triple, which isn't + # snapshotted onto the task (rotating a base_url or key should take + # effect immediately, unlike the experimental conditions). + profile = crud.get_agent_profile(db, task.agent_profile) + base_url = profile.base_url if profile else None + api_key_ref = profile.api_key_ref if profile else None + # Snapshot values needed after the session closes. + task_snapshot = { + "agent_profile": task.agent_profile, + "model": task.model, + "temperature": task.temperature, + "tools_json": task.tools_json, + "framework_version": task.framework_version + or (profile.framework_version if profile else None), + } + finally: + db.close() + + logging.info( + f"[Agent/inference] task validated — profile={task_snapshot['agent_profile']} " + f"runtime={task_snapshot['framework_version']} " + f"content={content_included}" + ) + return await inference.run_inference( + task_uuid=body.task_id, + session_uuid=session_id, + openai_body=body.request, + enrichment=body.enrichment, + agent_profile=task_snapshot["agent_profile"], + model=task_snapshot["model"], + temperature=task_snapshot["temperature"], + base_url=base_url, + api_key_ref=api_key_ref, + framework_version=task_snapshot["framework_version"], + profile_tools_json=task_snapshot["tools_json"], + content_included=content_included, + app=app, + ) + + +# ── Telemetry upload ───────────────────────────────────────────────────────── + +# Maps OTel span names produced by the plugin's trace collectors to the +# event_type vocabulary already used in agent_event rows. +_SPAN_EVENT_TYPE: dict[str, str] = { + "agent.llm.invoke": "model_call", + "agent.tool.execute": "tool_call", + "agent.retrieval": "observation", +} + + +def _safe_int(value: Any) -> Optional[int]: + """Coerce an OTel attribute value to int, or None if it isn't numeric.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + +class SpanPayload(BaseModel): + trace_id: str + span_id: str + parent_span_id: Optional[str] = None + name: str + kind: str = "INTERNAL" + start_time: str + end_time: str + duration_ms: int = 0 + attributes: dict[str, Any] = Field(default_factory=dict) + status: str = "OK" + + +class TelemetryUploadRequest(BaseModel): + trace_id: str + agent_runtime: str + experiment_tags: dict[str, str] = Field(default_factory=dict) + spans: list[SpanPayload] + # NOTE: no `content_included` field. Whether content is stored is resolved + # server-side from the user's preference; a client-supplied flag here would + # be exactly the client-trusting design merge decision 3 rejects. + + +@router.post("/task/{task_id}/telemetry", status_code=204) +def upload_agent_telemetry( + task_id: uuid.UUID, + body: TelemetryUploadRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Persist OTel spans for one completed agent session. + + Span → DB mapping:: + + agent.task root span → updates AgentTask aggregates + agent.llm.invoke child span → AgentEvent(event_type='model_call') + agent.tool.execute child span → AgentEvent(event_type='tool_call') + agent.retrieval child span → AgentEvent(event_type='observation') + + Known span attributes (token counts, model, tool name) map onto columns; + the rest of the attribute bag goes into ``extra_json``. + + This is a bulk path for runtimes whose calls the proxy didn't see + individually. Where both fire for the same session, the proxy's per-call + rows are the more detailed record; these spans mainly recover the + task-level totals. + """ + logging.info( + f"[Agent/telemetry] upload — task={task_id} trace={body.trace_id} " + f"runtime={body.agent_runtime} spans={len(body.spans)}" + ) + + db = app.get_db_session() + try: + _require_own_task(db, task_id, session_id) + + root = next((s for s in body.spans if s.parent_span_id is None), None) + child_spans = [s for s in body.spans if s.parent_span_id is not None] + + if root is not None: + llm_spans = [s for s in child_spans if s.name == "agent.llm.invoke"] + total_prompt = sum( + _safe_int(s.attributes.get("llm.prompt_tokens")) or 0 + for s in llm_spans + ) + total_completion = sum( + _safe_int(s.attributes.get("llm.completion_tokens")) or 0 + for s in llm_spans + ) + crud.update_agent_task_status( + db, + task_id=task_id, + status="done", + total_steps=len(child_spans), + input_tokens=total_prompt or None, + output_tokens=total_completion or None, + ) + logging.info( + f"[Agent/telemetry] task {task_id} updated — " + f"steps={len(child_spans)} tokens={total_prompt}/{total_completion}" + ) + + base_index = crud.count_agent_events_for_task(db, task_id) + for offset, span in enumerate(child_spans): + event_type = _SPAN_EVENT_TYPE.get(span.name, "observation") + attrs = span.attributes or {} + # OTel span ids are hex strings; only persist a parent link when it + # parses as a UUID, since that's the column type. + parent_span_uuid: Optional[uuid.UUID] = None + if span.parent_span_id: + try: + parent_span_uuid = uuid.UUID(str(span.parent_span_id)) + except (ValueError, TypeError): + parent_span_uuid = None + + crud.append_agent_event( + db, + task_id=task_id, + event_index=base_index + offset, + event_type=event_type, + source="proxy", + latency_ms=span.duration_ms if span.duration_ms > 0 else None, + span_id=span.span_id, + parent_span_id=parent_span_uuid, + model=attrs.get("llm.model") or attrs.get("model"), + prompt_tokens=_safe_int(attrs.get("llm.prompt_tokens")), + completion_tokens=_safe_int(attrs.get("llm.completion_tokens")), + total_tokens=_safe_int(attrs.get("llm.total_tokens")), + tool_name=( + attrs.get("tool.name") if event_type == "tool_call" else None + ), + extra_json=_span_extra_json(span, body), + commit=False, + ) + db.commit() + logging.info( + f"[Agent/telemetry] stored {len(child_spans)} events for task {task_id}" + ) + return Response(status_code=204) + + except HTTPException: + raise + except Exception as e: + db.rollback() + logging.error( + f"[Agent/telemetry] DB error for task {task_id} — {e}", exc_info=True + ) + raise HTTPException(status_code=500, detail="Failed to store telemetry") + finally: + db.close() + + +def _span_extra_json(span: SpanPayload, body: TelemetryUploadRequest) -> str: + """Serialise the non-columnar span metadata into the JSON overflow column. + + Group-21's version dropped these attributes on the floor. Keeping them in + ``extra_json`` is the point of the overflow column: an unrecognised span + attribute costs a nested JSON key rather than being lost. + """ + import json as _json + + known = { + "llm.model", + "model", + "llm.prompt_tokens", + "llm.completion_tokens", + "llm.total_tokens", + "tool.name", + } + return _json.dumps( + { + "otel": { + "trace_id": span.trace_id, + "span_name": span.name, + "kind": span.kind, + "status": span.status, + "start_time": span.start_time, + "end_time": span.end_time, + # Raw parent id, since the column can only hold UUID-shaped ones. + "parent_span_id": span.parent_span_id, + }, + "agent_runtime": body.agent_runtime, + "experiment_tags": body.experiment_tags or None, + "attributes": { + k: v for k, v in (span.attributes or {}).items() if k not in known + } + or None, + }, + default=str, + ) + + +# ── Task close ─────────────────────────────────────────────────────────────── + + +@router.post("/task/{task_id}/close", status_code=204) +def close_agent_task( + task_id: uuid.UUID, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Mark a task done and aggregate its telemetry into task-level totals. + + Called by the plugin on startup, closing the *previous* session's task, + because the plugin can't reliably detect when an ACP agent process exits. + Idempotent — safe to call repeatedly. + """ + db = app.get_db_session() + try: + _require_own_task(db, task_id, session_id) + summary = lifecycle.finalize_agent_task(db, task_id) + logging.info(f"[Agent/close] task {task_id} closed — {summary}") + return Response(status_code=204) + except HTTPException: + raise + except Exception as e: + db.rollback() + logging.error(f"[Agent/close] DB error for task {task_id} — {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Failed to close task") + finally: + db.close() diff --git a/src/backend/routers/analytics/studies.py b/src/backend/routers/analytics/studies.py index 6449dda..17a870b 100644 --- a/src/backend/routers/analytics/studies.py +++ b/src/backend/routers/analytics/studies.py @@ -32,9 +32,16 @@ class CreateStudyRequest(BaseModel): name: str description: Optional[str] = None starts_at: str # ISO datetime - ends_at: Optional[str] = None # ISO datetime + ends_at: Optional[str] = None # ISO datetime config_ids: List[int] # List of configuration IDs to test default_config_id: int # Configuration to use when study ends + # Optional agent-profile arms. Empty → a completion-only study, behaving + # exactly as before this field existed (backwards compatible). + agent_profile_ids: List[str] = [] # agent_profile UUIDs acting as A/B arms + # Which arm agent uplift is measured against; must be one of + # agent_profile_ids when set. Analogous to default_config_id on the + # completion side. + baseline_agent_profile_id: Optional[str] = None class UpdateStudyRequest(BaseModel): @@ -77,7 +84,37 @@ def create_study( config = crud.get_config_by_id(db_session, config_id) if not config: raise HTTPException(status_code=400, detail=f"Configuration {config_id} not found") - + + # Validate the optional agent-profile arms. Done before any writes so a + # bad arm list can't leave a half-configured study behind. + agent_profile_uuids: List[uuid.UUID] = [] + baseline_profile_uuid: Optional[uuid.UUID] = None + try: + agent_profile_uuids = [ + uuid.UUID(pid) for pid in study_request.agent_profile_ids + ] + if study_request.baseline_agent_profile_id is not None: + baseline_profile_uuid = uuid.UUID( + study_request.baseline_agent_profile_id + ) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid agent profile ID format") + + for profile_uuid in agent_profile_uuids: + if crud.get_agent_profile_by_id(db_session, profile_uuid) is None: + raise HTTPException( + status_code=400, + detail=f"Agent profile {profile_uuid} not found", + ) + if ( + baseline_profile_uuid is not None + and baseline_profile_uuid not in agent_profile_uuids + ): + raise HTTPException( + status_code=400, + detail="Baseline agent profile must be one of the selected profiles", + ) + # Check if there's already an active study active_study_query = """ SELECT study_id FROM study WHERE is_active = true LIMIT 1 @@ -115,15 +152,33 @@ def create_study( # If study is active, assign configurations to users if is_active: assign_users_to_study(db_session, study_id, study_request.config_ids) - + + # Attach the agent-profile arms. Persisted regardless of active state: + # only an *active* study's arms are drawn from + # (crud.list_active_study_agent_profiles), and agent assignment is + # sticky-lazy — drawn on a user's first agent task rather than + # pre-assigned like completion configs — so there's nothing to backfill + # here. + if agent_profile_uuids: + crud.set_study_agent_profiles( + db_session, + study_id=study_id, + profile_ids=agent_profile_uuids, + baseline_profile_id=baseline_profile_uuid, + ) + db_session.commit() - + return JsonResponseWithStatus( status_code=201, content={ "study_id": str(study_id), "name": study_request.name, "is_active": is_active, + "agent_profile_ids": [str(p) for p in agent_profile_uuids], + "baseline_agent_profile_id": ( + str(baseline_profile_uuid) if baseline_profile_uuid else None + ), "message": "Study created successfully" } ) @@ -326,6 +381,22 @@ def get_study_details( "engagement_rate": float(assignment.engagement_rate) if assignment.engagement_rate else 0.0 }) + # Agent-profile arms attached to this study. Empty for completion-only + # studies, which is the backwards-compatible default. + agent_profiles = [ + { + "profile_id": str(link.profile_id), + "name": link.profile.name if link.profile else None, + "model": link.profile.model if link.profile else None, + "framework_version": ( + link.profile.framework_version if link.profile else None + ), + "is_active": link.profile.is_active if link.profile else None, + "is_baseline": link.is_baseline, + } + for link in crud.list_study_agent_profiles(db_session, study_uuid) + ] + return JsonResponseWithStatus( status_code=200, content={ @@ -340,7 +411,8 @@ def get_study_details( "default_config_id": study.default_config_id, "created_at": study.created_at.isoformat() }, - "assignments": assignment_data + "assignments": assignment_data, + "agent_profiles": agent_profiles } ) @@ -624,3 +696,250 @@ def evaluate_study( finally: db_session.close() + + +@router.get("/{study_id}/agent-evaluation") +def evaluate_study_agents( + study_id: str, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance) +): + """ + Evaluate an A/B study's *agent* arms. + + The agent-side mirror of ``evaluate_study``: aggregates agent telemetry + (``agent_task`` + ``agent_event``) per attached agent profile, within the + study's time window, and reports uplift against the baseline arm. + + A task counts toward an arm when its ``agent_profile`` name matches a study + profile and it was created inside ``[starts_at, ends_at]``. Matching by name + rather than profile_id is deliberate — the name is what gets snapshotted + onto the task, so historical tasks stay attributable even if the profile row + is later edited or deleted. + + Because both telemetry paths write into the same ``agent_event`` table, the + same aggregation works whether an arm ran Goose, Codex, or the built-in + ``code4me2-agent`` runtime — which is the whole point of the unified schema. + + Admin only endpoint. + """ + db_session = app.get_db_session() + + try: + study_uuid = uuid.UUID(study_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid study ID format") + + try: + study_query = """ + SELECT study_id, name, starts_at, ends_at + FROM study WHERE study_id = :study_id + """ + study = db_session.execute( + text(study_query), {"study_id": study_uuid} + ).fetchone() + if not study: + raise HTTPException(status_code=404, detail="Study not found") + + window = { + "study_id": study_uuid, + "starts_at": study.starts_at, + "ends_at": study.ends_at, + } + + # Per-arm task-level aggregates. LEFT JOIN so an arm with zero tasks + # still appears as a row with zeros, rather than vanishing from the + # comparison (an arm nobody used is itself a finding). + task_query = """ + SELECT + ap.profile_id, + ap.name AS profile_name, + ap.model, + ap.framework_version, + sap.is_baseline, + COUNT(DISTINCT t.task_id) AS total_tasks, + COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(DISTINCT COALESCE(t.owner_user_id, t.session_id)) AS total_participants, + AVG(t.total_steps) AS avg_steps, + SUM(t.input_tokens) AS total_input_tokens, + SUM(t.output_tokens) AS total_output_tokens + FROM study_agent_profile sap + JOIN agent_profile ap ON ap.profile_id = sap.profile_id + LEFT JOIN agent_task t + ON t.agent_profile = ap.name + AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) + WHERE sap.study_id = :study_id + GROUP BY ap.profile_id, ap.name, ap.model, ap.framework_version, sap.is_baseline + ORDER BY ap.name + """ + task_rows = db_session.execute(text(task_query), window).fetchall() + + # Per-arm event-level aggregates, kept as a separate query so the task + # aggregates above aren't inflated by the event-row fan-out. + event_query = """ + SELECT + ap.name AS profile_name, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, + AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call') AS avg_model_latency_ms, + SUM(e.total_tokens) AS total_tokens + FROM study_agent_profile sap + JOIN agent_profile ap ON ap.profile_id = sap.profile_id + LEFT JOIN agent_task t + ON t.agent_profile = ap.name + AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) + LEFT JOIN agent_event e ON e.task_id = t.task_id + WHERE sap.study_id = :study_id + GROUP BY ap.name + """ + event_rows = { + row.profile_name: row + for row in db_session.execute(text(event_query), window).fetchall() + } + + # Human-in-the-loop signal: what the developer actually did with the + # agent's proposals. This is the metric group-5 couldn't report at all, + # and it's arguably the most important one — an arm that proposes more + # edits but gets fewer accepted is worse, not better. + edit_query = """ + SELECT + ap.name AS profile_name, + COUNT(ed.edit_id) AS total_edits, + COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS TRUE) AS accepted_edits, + COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS FALSE) AS rejected_edits, + COUNT(ed.edit_id) FILTER (WHERE ed.was_modified IS TRUE) AS modified_edits + FROM study_agent_profile sap + JOIN agent_profile ap ON ap.profile_id = sap.profile_id + LEFT JOIN agent_task t + ON t.agent_profile = ap.name + AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) + LEFT JOIN agent_edit ed ON ed.task_id = t.task_id + WHERE sap.study_id = :study_id + GROUP BY ap.name + """ + edit_rows = { + row.profile_name: row + for row in db_session.execute(text(edit_query), window).fetchall() + } + + def _f(value) -> float: + return float(value) if value is not None else 0.0 + + def _i(value) -> int: + return int(value) if value is not None else 0 + + arm_results = [] + baseline_arm = None + + for row in task_rows: + events = event_rows.get(row.profile_name) + edits = edit_rows.get(row.profile_name) + total_tasks = _i(row.total_tasks) + total_edits = _i(edits.total_edits) if edits else 0 + accepted_edits = _i(edits.accepted_edits) if edits else 0 + + arm = { + "profile_id": str(row.profile_id), + "profile_name": row.profile_name, + "model": row.model, + "framework_version": row.framework_version, + "is_baseline": bool(row.is_baseline), + "metrics": { + "total_tasks": total_tasks, + "completed_tasks": _i(row.completed_tasks), + "completion_rate": _i(row.completed_tasks) / max(total_tasks, 1), + "total_participants": _i(row.total_participants), + "avg_steps": _f(row.avg_steps), + "total_input_tokens": _i(row.total_input_tokens), + "total_output_tokens": _i(row.total_output_tokens), + "model_calls": _i(events.model_calls) if events else 0, + "tool_calls": _i(events.tool_calls) if events else 0, + "avg_model_latency_ms": ( + _f(events.avg_model_latency_ms) if events else 0.0 + ), + "total_tokens": _i(events.total_tokens) if events else 0, + "total_edits": total_edits, + "accepted_edits": accepted_edits, + "rejected_edits": _i(edits.rejected_edits) if edits else 0, + "modified_edits": _i(edits.modified_edits) if edits else 0, + # Undecided edits are excluded from the denominator: an edit + # the developer hasn't ruled on yet is not a rejection. + "edit_acceptance_rate": ( + accepted_edits + / max( + accepted_edits + (_i(edits.rejected_edits) if edits else 0), + 1, + ) + ), + }, + } + + if arm["is_baseline"]: + baseline_arm = arm + arm_results.append(arm) + + # Uplift vs the baseline arm, when one is designated. + if baseline_arm: + base = baseline_arm["metrics"] + for arm in arm_results: + if arm["is_baseline"]: + continue + metrics = arm["metrics"] + arm["uplift"] = { + "completion_rate_change_pct": _pct_change( + metrics["completion_rate"], base["completion_rate"] + ), + "edit_acceptance_change_pct": _pct_change( + metrics["edit_acceptance_rate"], base["edit_acceptance_rate"] + ), + "avg_steps_change_pct": _pct_change( + metrics["avg_steps"], base["avg_steps"] + ), + "latency_change_pct": _pct_change( + metrics["avg_model_latency_ms"], base["avg_model_latency_ms"] + ), + "is_better_completion": ( + metrics["completion_rate"] > base["completion_rate"] + ), + "is_better_acceptance": ( + metrics["edit_acceptance_rate"] > base["edit_acceptance_rate"] + ), + "is_faster": ( + metrics["avg_model_latency_ms"] < base["avg_model_latency_ms"] + ), + } + + return JsonResponseWithStatus( + status_code=200, + content={ + "study": { + "study_id": str(study.study_id), + "name": study.name, + "starts_at": study.starts_at.isoformat(), + "ends_at": study.ends_at.isoformat() if study.ends_at else None, + }, + "results": arm_results, + }, + ) + + except HTTPException: + raise + except Exception as e: + db_session.rollback() + raise HTTPException( + status_code=500, detail=f"Error evaluating study agents: {str(e)}" + ) + finally: + db_session.close() + + +def _pct_change(value: float, baseline: float) -> float: + """Percentage change of ``value`` against ``baseline``. + + Returns 0.0 when the baseline is zero: with no baseline to compare against, + a made-up ratio would read as a real effect. The absolute metrics are + reported alongside, so a caller can see that the baseline arm had no data. + """ + if not baseline: + return 0.0 + return ((value - baseline) / baseline) * 100 diff --git a/src/backend/routers/session/deactivate.py b/src/backend/routers/session/deactivate.py index 2df1298..65676c0 100644 --- a/src/backend/routers/session/deactivate.py +++ b/src/backend/routers/session/deactivate.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Cookie, Depends +from agents import lifecycle from App import App from backend.Responses import ( DeactivateSessionError, @@ -90,6 +91,22 @@ def deactivate_session( status_code=401, content=InvalidOrExpiredSessionToken(), ) + # Close any agent tasks this session left open before tearing it down. + # The plugin closes the *previous* task on startup, but it can't detect + # logout, so without this a task ends its life stuck in 'running' with + # no aggregated totals. Failures here are logged and ignored: telemetry + # bookkeeping must not block the user from logging out. + for task in crud.get_open_agent_tasks_for_session( + db_session, uuid.UUID(session_token) + ): + try: + lifecycle.finalize_agent_task(db_session, task.task_id) + except Exception as e: + logging.warning( + f"[Session/deactivate] failed to finalize agent task " + f"{task.task_id}: {e}" + ) + # Step 3: Delete the session from Redis and clear cookies redis_manager.delete("user_token", user_id, db_session) redis_manager.delete("session_token", session_token, db_session) diff --git a/src/code4me2_agent/__init__.py b/src/code4me2_agent/__init__.py new file mode 100644 index 0000000..4c7d33d --- /dev/null +++ b/src/code4me2_agent/__init__.py @@ -0,0 +1,16 @@ +from code4me2_agent.adapters import AdapterResult, AgentAdapter, create_agent_adapter +from code4me2_agent.command_tools import WorkspaceCommandTools +from code4me2_agent.config import AgentConfig +from code4me2_agent.echo import EchoAgentCore, EchoPromptResult +from code4me2_agent.file_tools import WorkspaceFileTools + +__all__ = [ + "AdapterResult", + "AgentAdapter", + "AgentConfig", + "EchoAgentCore", + "EchoPromptResult", + "WorkspaceFileTools", + "WorkspaceCommandTools", + "create_agent_adapter", +] diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py new file mode 100644 index 0000000..0283168 --- /dev/null +++ b/src/code4me2_agent/acp_runtime.py @@ -0,0 +1,856 @@ +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from code4me2_agent.acp_updates import AcpUpdateBuilder +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import EventLoopAsyncRunner +from code4me2_agent.command_tools import build_acp_command_backend +from code4me2_agent.echo import EchoAgentCore +from code4me2_agent.file_tools import build_acp_file_system_backend +from code4me2_agent.runtime_auth import AcpAuthorizationFailure, AcpBackendAuthorization + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from acp.interfaces import Client + + from code4me2_agent.config import AgentConfig, CommandConfig + from code4me2_agent.events import ToolCallEvent + from code4me2_agent.telemetry import AgentTelemetryRecorder + + +def _prompt_text(prompt: list[Any]) -> str: + parts: list[str] = [] + for block in prompt: + if isinstance(block, dict): + block_type = block.get("type", "") + text = block.get("text", "") + if block_type == "resource_link": + text = _resource_link_text(block) + else: + block_type = getattr(block, "type", "") + text = getattr(block, "text", "") + if block_type == "resource_link": + text = _resource_link_text(block) + if text: + parts.append(text) + return "\n".join(parts) + + +async def _prompt_text_async(prompt: list[Any]) -> str: + parts: list[str] = [] + for block in prompt: + if isinstance(block, dict): + block_type = block.get("type", "") + text = block.get("text", "") + if block_type == "resource_link": + text = await _resource_link_text_async(block) + else: + block_type = getattr(block, "type", "") + text = getattr(block, "text", "") + if block_type == "resource_link": + text = await _resource_link_text_async(block) + if text: + parts.append(text) + return "\n".join(parts) + + +def _resource_link_text(block: object) -> str: + uri = str(capability_value(block, "uri") or "").strip() + name = str(capability_value(block, "name") or uri or "resource").strip() + details = [] + mime_type = capability_value(block, "mimeType") or capability_value( + block, "mime_type" + ) + description = capability_value(block, "description") + if mime_type: + details.append(f"mimeType={mime_type}") + if description: + details.append(f"description={description}") + suffix = f"; {'; '.join(details)}" if details else "" + resource_link = f"[Resource link: {name} <{uri}>{suffix}]" + return resource_link + + +async def _resource_link_text_async(block: object) -> str: + uri = str(capability_value(block, "uri") or "").strip() + name = str(capability_value(block, "name") or uri or "resource").strip() + details = [] + mime_type = capability_value(block, "mimeType") or capability_value( + block, "mime_type" + ) + description = capability_value(block, "description") + if mime_type: + details.append(f"mimeType={mime_type}") + if description: + details.append(f"description={description}") + suffix = f"; {'; '.join(details)}" if details else "" + resource_link = f"[Resource link: {name} <{uri}>{suffix}]" + return resource_link + +def _acp_stop_reason(adapter_stop_reason: str) -> str: + if adapter_stop_reason == "max_iterations": + return "max_turn_requests" + if adapter_stop_reason in { + "end_turn", + "max_tokens", + "max_turn_requests", + "refusal", + "cancelled", + }: + return adapter_stop_reason + return "refusal" + + +@dataclass +class AgentSession: + session_id: str + core: EchoAgentCore + + +class AcpSessionEventSink: + def __init__( + self, + *, + conn: Client, + session_id: str, + updates: AcpUpdateBuilder, + telemetry: object, + async_runner: EventLoopAsyncRunner, + ) -> None: + self._conn = conn + self._session_id = session_id + self._updates = updates + self._telemetry = telemetry + self._async_runner = async_runner + + def tool_call(self, event: ToolCallEvent) -> None: + if event.phase == "started": + update = self._updates.start_tool_call( + tool_call_id=event.tool_call_id, + title=event.title, + kind=event.kind, + status=event.status, + path=event.path, + raw_input=event.raw_input, + ) + else: + content = ( + [self._updates.text_tool_content(event.content_text)] + if event.content_text + else None + ) + update = self._updates.update_tool_call( + tool_call_id=event.tool_call_id, + title=event.title, + kind=event.kind, + status=event.status, + content=content, + raw_output=event.raw_output, + ) + self._send_update(event=event, update=update) + + def thought(self, event: object) -> None: + phase = getattr(event, "phase", None) + duration_ms = getattr(event, "duration_ms", None) + timing_metadata = None + if phase is not None: + phase_metadata: dict[str, object] = {"phase": phase} + if duration_ms is not None: + phase_metadata["durationMs"] = duration_ms + timing_metadata = {"code4me2": phase_metadata} + update = self._updates.agent_thought( + getattr(event, "text", ""), + metadata=timing_metadata, + ) + try: + self._async_runner.run( + self._conn.session_update( + session_id=self._session_id, + update=update, + source="code4me2_agent", + ) + ) + except Exception as exc: + record = getattr(self._telemetry, "record", None) + if callable(record): + record( + event_type="agent.acp.update_failed", + run_id=getattr(event, "run_id", None), + request_id=getattr(event, "request_id", None), + parent_event_id=None, + payload={ + "update_type": "agent_thought", + "error_message": str(exc), + }, + ) + + def _send_update(self, *, event: ToolCallEvent, update: object) -> None: + try: + self._async_runner.run( + self._conn.session_update( + session_id=self._session_id, + update=update, + source="code4me2_agent", + ) + ) + except Exception as exc: + record = getattr(self._telemetry, "record", None) + if callable(record): + record( + event_type="agent.acp.update_failed", + run_id=event.run_id, + request_id=event.request_id, + parent_event_id=None, + payload={ + "tool_call_id": event.tool_call_id, + "tool_name": event.tool_name, + "phase": event.phase, + "error_message": str(exc), + }, + ) + + +def _record_acp_runtime_event( + telemetry: AgentTelemetryRecorder, + *, + event_type: str, + session_id: str, + request_id: str, + payload: dict[str, Any], +) -> None: + telemetry.record( + event_type=event_type, + run_id=f"acp-runtime-{request_id}", + request_id=request_id, + parent_event_id=None, + payload={"session_id": session_id, **payload}, + ) + + +def _normalize_capabilities(value: object | None) -> object | None: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _normalize_capabilities(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_normalize_capabilities(item) for item in value] + if hasattr(value, "model_dump"): + dump = getattr(value, "model_dump") + if callable(dump): + return _normalize_capabilities(dump(by_alias=True, exclude_none=True)) + if hasattr(value, "__dict__"): + return { + str(key): _normalize_capabilities(item) + for key, item in vars(value).items() + if not str(key).startswith("_") + } + return repr(value) + + +def _prompt_backend_state(session: AgentSession) -> dict[str, Any]: + file_backend = getattr(session.core.file_tools, "_acp_backend", None) + command_backend = getattr(session.core.command_tools, "_acp_backend", None) + return { + "workspace_root": session.core._config.workspace_root.as_posix(), + "file_backend": "acp" if file_backend is not None else "local", + "file_backend_session_id": getattr(file_backend, "session_id", None), + "read_text_file_enabled": bool( + getattr(file_backend, "read_text_file_enabled", False) + ), + "write_text_file_enabled": bool( + getattr(file_backend, "write_text_file_enabled", False) + ), + "command_backend": "acp" if command_backend is not None else "local", + "terminal_enabled": bool(getattr(command_backend, "terminal_enabled", False)), + } + + +def _persisted_memory_messages(payload: dict) -> list[dict[str, Any]]: + messages = payload.get("messages", []) + if not isinstance(messages, list): + return [] + return [message for message in messages if isinstance(message, dict)] + + +def _should_record_runtime_context( + *, + client_capabilities: object | None, + session_lookup: str = "existing", +) -> bool: + return client_capabilities is not None or session_lookup != "existing" + + +def create_acp_agent(config: AgentConfig, *, authorization: Any | None = None) -> Any: + # TODO new agent -> a lot of changes to here. + try: + from acp import ( + Agent, + AuthenticateResponse, + InitializeResponse, + LoadSessionResponse, + NewSessionResponse, + PromptResponse, + RequestError, + ) + from acp.schema import ( + AgentCapabilities, + AuthMethod, + Implementation, + McpCapabilities, + PromptCapabilities, + ResumeSessionResponse, + SessionCapabilities, + SessionResumeCapabilities, + ) + except ImportError as exc: + raise SystemExit( + "The ACP SDK is required to run stdio mode. " + "Install this package from source with agent-client-protocol available." + ) from exc + + class Code4MeEchoAgent(Agent): + _conn: Client + _client_capabilities: object | None + _updates: AcpUpdateBuilder + _bootstrap_core: EchoAgentCore + _sessions: dict[str, AgentSession] + _cancelled_sessions: set[str] + _authorization: Any + _authorized_workspace: Path | None + + def on_connect(self, conn: Client) -> None: + self._conn = conn + self._client_capabilities = None + self._updates = AcpUpdateBuilder() + self._sessions = {} + self._cancelled_sessions = set() + self._authorization = ( + authorization + or AcpBackendAuthorization.from_environment( + workspace_root=config.workspace_root, + ) + ) + self._authorized_workspace = None + self._current_config = config + self._bootstrap_core = EchoAgentCore(config) + self.file_tools = self._bootstrap_core.file_tools + self.command_tools = self._bootstrap_core.command_tools + + def _build_session( + self, *, session_config: AgentConfig, session_id: str + ) -> AgentSession: + async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) + event_sink = AcpSessionEventSink( + conn=self._conn, + session_id=session_id, + updates=self._updates, + telemetry=None, + async_runner=async_runner, + ) + core = EchoAgentCore(session_config, event_sink=event_sink) + event_sink._telemetry = core._telemetry + core.rebuild_tools( + acp_file_backend=build_acp_file_system_backend( + client=self._conn, + session_id=session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ), + acp_command_backend=build_acp_command_backend( + client=self._conn, + session_id=session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ), + ) + return AgentSession(session_id=session_id, core=core) + + def _load_persisted_memory(self, session: AgentSession) -> None: + request_json = getattr(self._authorization, "authorized_json_request", None) + if not callable(request_json): + return + try: + payload = request_json( + "GET", + f"/api/agent/memory/{session.session_id}", + None, + ) + except AcpAuthorizationFailure as exc: + logger.info("Could not load persisted ACP memory: %s", exc) + return + messages = _persisted_memory_messages(payload) + if messages: + session.core.load_session_memory(messages) + + def _save_persisted_memory(self, session: AgentSession) -> None: + snapshot = session.core.session_memory_snapshot() + if snapshot is None: + return + request_json = getattr(self._authorization, "authorized_json_request", None) + if not callable(request_json): + return + try: + request_json( + "PUT", + f"/api/agent/memory/{session.session_id}", + {"messages": snapshot}, + ) + except AcpAuthorizationFailure as exc: + logger.info("Could not save persisted ACP memory: %s", exc) + + def _session_config( + self, *, workspace_root: Path, session_id: str + ) -> AgentConfig: + # Build from self._current_config, NOT the module-level `config` + # closure: _apply_server_config writes the assigned agent profile + # into _current_config, and reading the original file config here + # would silently drop the server's model/provider/tool overrides + # for every session created after authentication — i.e. the A/B + # assignment would appear to work (it's logged) but never actually + # take effect on a new session. + base_config = self._current_config + telemetry_headers = self._authorization.telemetry_headers() + provider_auth_headers = ( + telemetry_headers + if base_config.adapter.provider.kind == "code4me_backend" + else base_config.adapter.provider.auth_headers + ) + upload_config = base_config.upload + backend_url = getattr(self._authorization, "backend_url", None) + if backend_url and upload_config.ingest_url is None: + # Self-reported telemetry converges on the same agent_event + # table the inference relay writes to (merge decision 2). + upload_config = replace( + upload_config, + enabled=True, + ingest_url=f"{str(backend_url).rstrip('/')}/api/agent/events/ingest", + ) + return replace( + base_config, + workspace_root=workspace_root, + session_id=session_id, + upload=replace(upload_config, auth_headers=telemetry_headers), + adapter=replace( + base_config.adapter, + provider=replace( + base_config.adapter.provider, + auth_headers=provider_auth_headers, + ), + ), + ) + + async def _create_or_load_session( + self, + *, + cwd: str, + session_id: str, + event_type: str, + ) -> AgentSession: + await self._require_authenticated() + workspace_root = _resolve_session_cwd(cwd) + if workspace_root != self._authorized_workspace: + raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + session_config = self._session_config( + workspace_root=workspace_root, + session_id=session_id, + ) + session = self._build_session( + session_config=session_config, session_id=session_id + ) + await asyncio.to_thread(self._load_persisted_memory, session) + self._sessions[session_id] = session + self.file_tools = session.core.file_tools + self.command_tools = session.core.command_tools + if _should_record_runtime_context( + client_capabilities=self._client_capabilities + ): + _record_acp_runtime_event( + session.core._telemetry, + event_type=event_type, + session_id=session_id, + request_id=uuid4().hex, + payload={ + "cwd": session_config.workspace_root.as_posix(), + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + **_prompt_backend_state(session), + }, + ) + return session + + async def initialize( + self, + protocol_version: int, + client_capabilities: object | None = None, + **kwargs: Any, + ) -> InitializeResponse: + self._client_capabilities = client_capabilities or kwargs.get( + "clientCapabilities" + ) + async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) + acp_file_backend = build_acp_file_system_backend( + client=self._conn, + session_id=config.session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ) + acp_command_backend = build_acp_command_backend( + client=self._conn, + session_id=config.session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ) + self._bootstrap_core.rebuild_tools( + acp_file_backend=acp_file_backend, + acp_command_backend=acp_command_backend, + ) + self.file_tools = self._bootstrap_core.file_tools + self.command_tools = self._bootstrap_core.command_tools + initialize_auth_succeeded = False + try: + await self._authenticate_if_needed() + initialize_auth_succeeded = True + except RequestError: + # Some ACP clients never invoke the custom auth handshake for local agents. + # Try once during initialize, but keep later session/prompt calls fail-closed. + pass + _record_acp_runtime_event( + self._bootstrap_core._telemetry, + event_type="agent.acp.initialize", + session_id=config.session_id, + request_id=uuid4().hex, + payload={ + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + "authorization_bootstrap_succeeded": initialize_auth_succeeded, + "file_backend": "acp" if acp_file_backend is not None else "local", + "command_backend": ( + "acp" if acp_command_backend is not None else "local" + ), + "read_text_file_enabled": bool( + getattr(acp_file_backend, "read_text_file_enabled", False) + ), + "write_text_file_enabled": bool( + getattr(acp_file_backend, "write_text_file_enabled", False) + ), + "terminal_enabled": bool( + getattr(acp_command_backend, "terminal_enabled", False) + ), + }, + ) + return InitializeResponse( + protocol_version=protocol_version, + agent_capabilities=AgentCapabilities( + loadSession=True, + promptCapabilities=PromptCapabilities( + image=False, + audio=False, + embeddedContext=False, + ), + sessionCapabilities=SessionCapabilities( + resume=SessionResumeCapabilities(), + ), + mcpCapabilities=McpCapabilities( + http=False, + sse=False, + ), + ), + agent_info=Implementation( + name="code4me2-agent", + title="Code4Me Agent", + version="0.1.0", + ), + auth_methods=[ + AuthMethod( + id="code4me-plugin-session", + name="Code4Me Plugin Session", + description="Authenticate using a prepared Code4Me project session.", + ) + ], + ) + + async def authenticate( + self, method_id: str, **kwargs: Any + ) -> AuthenticateResponse: + if method_id != "code4me-plugin-session": + raise RequestError.auth_required({"reason": "unsupported_auth_method"}) + await self._authenticate_if_needed() + return AuthenticateResponse() + + async def _authenticate_if_needed(self, *, force: bool = False) -> None: + if ( + not force + and self._authorization.is_authenticated + and self._authorized_workspace is not None + ): + logger.info( + "ACP runtime already has authenticated workspace=%s.", + self._authorized_workspace, + ) + return + # Retry with delay: the plugin may need time to write a fresh ACP handoff + # after a full restart (Docker + IDE) before the agent can use it. + max_attempts = 5 + for attempt in range(1, max_attempts + 1): + try: + logger.info( + "ACP runtime starting backend authentication (attempt %d/%d).", + attempt, + max_attempts, + ) + scope = await asyncio.to_thread(self._authorization.authenticate) + except AcpAuthorizationFailure as exc: + if attempt < max_attempts: + logger.warning( + "ACP runtime backend authentication rejected (attempt %d/%d): %s.", + attempt, + max_attempts, + exc, + ) + await asyncio.sleep(2.0) + continue + logger.warning( + "ACP runtime backend authentication rejected after %d attempts: %s.", + max_attempts, + exc, + ) + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + break + self._authorized_workspace = Path(scope.workspace).resolve() + logger.info( + "ACP runtime authorized workspace=%s project_id=%s.", + self._authorized_workspace, + scope.project_id, + ) + self._apply_server_config() + + def _apply_server_config(self) -> None: + """Adopt the assigned agent profile handed down by the backend. + + The whole merge of provider settings lives in + ``AgentConfig.with_server_overrides`` so there's one place that + decides precedence (server wins, local file is fallback) and one + place that decides routing (direct provider vs backend relay). + """ + server_config = self._authorization.server_agent_config + if server_config is None or not server_config.has_overrides: + logger.info("No server agent config overrides to apply.") + return + + base = self._current_config + backend_url = getattr(self._authorization, "backend_url", None) + new_config = base.with_server_overrides( + server_config, backend_url=backend_url + ) + self._current_config = new_config + self._apply_config_to_all_cores(new_config) + + provider = new_config.adapter.provider + logger.info( + "Applied server agent config: profile=%s model=%s provider_kind=%s " + "base_url=%s max_iterations=%s commands_allowlist=%s.", + server_config.agent_profile, + provider.model, + provider.kind, + provider.base_url, + new_config.adapter.max_iterations, + new_config.commands.allowlisted_commands, + ) + if not provider.is_configured: + # Loud, because the alternative is a confusing failure deep in + # the first model call. + logger.warning( + "Agent provider is still incompletely configured " + "(base_url=%r model=%r) — model calls will fail until the " + "assigned profile supplies both.", + provider.base_url, + provider.model, + ) + + def _apply_config_to_all_cores(self, new_config: Any) -> None: + from code4me2_agent.command_tools import WorkspaceCommandTools + + self._bootstrap_core._config = new_config + self._bootstrap_core._adapter._config = new_config + self._bootstrap_core.command_tools._config = new_config + self._bootstrap_core.command_tools._allowlisted_commands = set( + new_config.commands.allowlisted_commands + ) + self._bootstrap_core.file_tools._config = new_config + for session_id, session in self._sessions.items(): + session.core._config = new_config + session.core._adapter._config = new_config + session.core.command_tools._config = new_config + session.core.command_tools._allowlisted_commands = set( + new_config.commands.allowlisted_commands + ) + session.core.file_tools._config = new_config + logger.info( + "Updated %d active session(s) with server agent config.", + len(self._sessions), + ) + + async def _require_authenticated(self) -> None: + if ( + not self._authorization.is_authenticated + or self._authorized_workspace is None + ): + await self._authenticate_if_needed() + if ( + not self._authorization.is_authenticated + or self._authorized_workspace is None + ): + raise RequestError.auth_required() + + async def new_session( + self, + cwd: str, + **kwargs: Any, + ) -> NewSessionResponse: + session_id = str( + kwargs.get("session_id") or kwargs.get("sessionId") or uuid4().hex + ) + await self._create_or_load_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_created", + ) + return NewSessionResponse(session_id=session_id) + + async def load_session( + self, + cwd: str, + session_id: str, + **kwargs: Any, + ) -> LoadSessionResponse: + await self._create_or_load_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_loaded", + ) + return LoadSessionResponse() + + async def resume_session( + self, + cwd: str, + session_id: str, + **kwargs: Any, + ) -> ResumeSessionResponse: + await self._create_or_load_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_resumed", + ) + return ResumeSessionResponse() + + async def cancel( + self, + session_id: str, + **kwargs: Any, + ) -> None: + await self._require_authenticated() + self._cancelled_sessions.add(session_id) + + async def prompt( + self, + prompt: list[Any], + session_id: str, + message_id: str | None = None, + **kwargs: Any, + ) -> PromptResponse: + await self._require_authenticated() + try: + await asyncio.to_thread(self._authorization.validate) + except AcpAuthorizationFailure: + self._authorized_workspace = None + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + request_id = message_id or uuid4().hex + if session_id in self._cancelled_sessions: + self._cancelled_sessions.discard(session_id) + return PromptResponse( + stop_reason="cancelled", + user_message_id=message_id, + ) + session = self._sessions.get(session_id) + if session is None: + raise RequestError.auth_required({"reason": "unknown_session"}) + if _should_record_runtime_context( + client_capabilities=self._client_capabilities, + session_lookup="existing", + ): + _record_acp_runtime_event( + session.core._telemetry, + event_type="agent.acp.prompt_context", + session_id=session_id, + request_id=request_id, + payload={ + "session_lookup": "existing", + "known_session_ids": sorted(self._sessions.keys()), + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + **_prompt_backend_state(session), + }, + ) + prompt_text = await _prompt_text_async(prompt) + # 1 + logger.info("Prompt text: %s", prompt_text) + result = await asyncio.to_thread( + session.core.handle_prompt, + prompt_text, + request_id, + message_id, + ) + await asyncio.to_thread(self._save_persisted_memory, session) + # n + await self._conn.session_update( + session_id=session_id, + update=self._updates.agent_message( + result.final_response, + metadata={ + "code4me2": { + "phase": "completed", + "durationMs": result.duration_ms, + } + }, + ), + source="code4me2_agent", + ) + return PromptResponse( + stop_reason=_acp_stop_reason(result.stop_reason), + user_message_id=message_id, + ) + + return Code4MeEchoAgent() + + +def _resolve_session_cwd(cwd: str) -> Path: + workspace_root = Path(cwd).expanduser() + if not workspace_root.is_absolute(): + raise ValueError("ACP session cwd must be an absolute path.") + return workspace_root.resolve() + + +async def run_acp_stdio(config: AgentConfig) -> None: + try: + from acp import run_agent + except ImportError as exc: + raise SystemExit( + "The ACP SDK is required to run stdio mode. " + "Install this package from source with agent-client-protocol available." + ) from exc + + await run_agent(create_acp_agent(config), use_unstable_protocol=True) diff --git a/src/code4me2_agent/acp_updates.py b/src/code4me2_agent/acp_updates.py new file mode 100644 index 0000000..152314d --- /dev/null +++ b/src/code4me2_agent/acp_updates.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version +from typing import Any, Sequence + + +class AcpUpdateBuilder: + """Small compatibility layer over the pinned ACP Python SDK update surface.""" + + def __init__(self) -> None: + import acp + from acp import schema + + self._acp = acp + self._schema = schema + + @property + def sdk_version(self) -> str: + try: + return version("agent-client-protocol") + except PackageNotFoundError: + return "unknown" + + @property + def helper_gaps(self) -> tuple[str, ...]: + required_helpers = ( + "text_block", + "tool_content", + "tool_diff_content", + "tool_terminal_ref", + "update_agent_message", + "update_agent_thought", + "start_tool_call", + "start_read_tool_call", + "start_edit_tool_call", + "update_tool_call", + ) + return tuple(name for name in required_helpers if not hasattr(self._acp, name)) + + def agent_message(self, text: str, *, metadata: dict[str, Any] | None = None) -> Any: + text_block = getattr(self._acp, "text_block", None) + update_agent_message = getattr(self._acp, "update_agent_message", None) + if text_block is not None and update_agent_message is not None: + update = update_agent_message(text_block(text)) + else: + update = self._schema.AgentMessageChunk( + session_update="agent_message_chunk", + content=self._schema.TextContentBlock(type="text", text=text), + ) + if metadata is None: + return update + return update.model_copy(update={"field_meta": metadata}) + + def agent_thought(self, text: str, *, metadata: dict[str, Any] | None = None) -> Any: + text_block = getattr(self._acp, "text_block", None) + update_agent_thought = getattr(self._acp, "update_agent_thought", None) + if text_block is not None and update_agent_thought is not None: + update = update_agent_thought(text_block(text)) + else: + update = self._schema.AgentThoughtChunk( + session_update="agent_thought_chunk", + content=self._schema.TextContentBlock(type="text", text=text), + ) + if metadata is None: + return update + return update.model_copy(update={"field_meta": metadata}) + + def text_tool_content(self, text: str) -> Any: + text_block = getattr(self._acp, "text_block", None) + tool_content = getattr(self._acp, "tool_content", None) + if text_block is not None and tool_content is not None: + return tool_content(text_block(text)) + return self._schema.ContentToolCallContent( + type="content", + content=self._schema.TextContentBlock(type="text", text=text), + ) + + def diff_tool_content(self, path: str, *, new_text: str, old_text: str | None = None) -> Any: + tool_diff_content = getattr(self._acp, "tool_diff_content", None) + if tool_diff_content is not None: + return tool_diff_content(path, new_text, old_text) + return self._schema.FileEditToolCallContent( + type="diff", + path=path, + old_text=old_text, + new_text=new_text, + ) + + def terminal_tool_content(self, terminal_id: str) -> Any: + tool_terminal_ref = getattr(self._acp, "tool_terminal_ref", None) + if tool_terminal_ref is not None: + return tool_terminal_ref(terminal_id) + return self._schema.TerminalToolCallContent( + type="terminal", + terminal_id=terminal_id, + ) + + def start_tool_call( + self, + *, + tool_call_id: str, + title: str, + kind: str, + status: str = "pending", + path: str | None = None, + content: Sequence[Any] | None = None, + raw_input: Any | None = None, + raw_output: Any | None = None, + ) -> Any: + locations = [self._schema.ToolCallLocation(path=path)] if path else None + if raw_input is None and path is not None: + raw_input = {"path": path} + + start_tool_call = getattr(self._acp, "start_tool_call", None) + if start_tool_call is not None: + return start_tool_call( + tool_call_id, + title, + kind=kind, + status=status, + content=content, + locations=locations, + raw_input=raw_input, + raw_output=raw_output, + ) + return self._schema.ToolCallStart( + session_update="tool_call", + tool_call_id=tool_call_id, + title=title, + kind=kind, + status=status, + content=list(content) if content is not None else None, + locations=locations, + raw_input=raw_input, + raw_output=raw_output, + ) + + def update_tool_call( + self, + *, + tool_call_id: str, + title: str | None = None, + kind: str | None = None, + status: str | None = None, + content: Sequence[Any] | None = None, + raw_input: Any | None = None, + raw_output: Any | None = None, + ) -> Any: + update_tool_call = getattr(self._acp, "update_tool_call", None) + if update_tool_call is not None: + return update_tool_call( + tool_call_id, + title=title, + kind=kind, + status=status, + content=content, + raw_input=raw_input, + raw_output=raw_output, + ) + return self._schema.ToolCallProgress( + session_update="tool_call_update", + tool_call_id=tool_call_id, + title=title, + kind=kind, + status=status, + content=list(content) if content is not None else None, + raw_input=raw_input, + raw_output=raw_output, + ) + + def permission_request( + self, + *, + session_id: str, + tool_call_id: str, + title: str, + kind: str, + ) -> Any: + return self._schema.RequestPermissionRequest( + session_id=session_id, + tool_call=self.update_tool_call( + tool_call_id=tool_call_id, + title=title, + kind=kind, + status="pending", + ), + options=[ + self._schema.PermissionOption( + option_id="allow_once", + name="Allow once", + kind="allow_once", + ), + self._schema.PermissionOption( + option_id="reject_once", + name="Reject once", + kind="reject_once", + ), + ], + ) diff --git a/src/code4me2_agent/acp_utils.py b/src/code4me2_agent/acp_utils.py new file mode 100644 index 0000000..c9d6bdb --- /dev/null +++ b/src/code4me2_agent/acp_utils.py @@ -0,0 +1,9 @@ +from __future__ import annotations + + +def capability_value(capabilities: object | None, name: str) -> object | None: + if capabilities is None: + return None + if isinstance(capabilities, dict): + return capabilities.get(name) + return getattr(capabilities, name, None) diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py new file mode 100644 index 0000000..42f016d --- /dev/null +++ b/src/code4me2_agent/adapters.py @@ -0,0 +1,1666 @@ +from __future__ import annotations + +import json +import os +import re +from dataclasses import asdict, dataclass, is_dataclass +from time import perf_counter, sleep +from typing import TYPE_CHECKING, Any, Protocol +from openai import APIError, APIStatusError, OpenAI +import logging +import fcntl +from pathlib import Path +from time import time + +if TYPE_CHECKING: + from code4me2_agent.command_tools import WorkspaceCommandTools + from code4me2_agent.config import AgentConfig + from code4me2_agent.events import AgentEventSink + from code4me2_agent.file_tools import WorkspaceFileTools + from code4me2_agent.telemetry import AgentTelemetryRecorder + + +@dataclass(frozen=True) +class AdapterResult: + final_response: str + stop_reason: str + run_status: str + thoughts: tuple[str, ...] = () + + +class AgentAdapter(Protocol): + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + ) -> AdapterResult: ... + + +@dataclass(frozen=True) +class ToolCall: + tool_call_id: str + name: str + arguments: dict[str, Any] + + +@dataclass(frozen=True) +class ParsedProviderOutput: + tool_calls: list[ToolCall] + final_answer: str | None + thought: str | None = None + + +@dataclass(frozen=True) +class ProviderTurn: + output: dict[str, Any] + usage: dict[str, int] + finish_reason: str | None + model: str + request_payload: dict[str, Any] | None = None + raw_response: dict[str, Any] | None = None + + +class FakeProviderExhaustedError(RuntimeError): + pass + + +class BackendProviderRequestError(RuntimeError): + pass + + +def _rate_limit_provider_request_from_env() -> None: + state_path_text = os.getenv("CODE4ME_RATE_LIMIT_STATE_PATH", "").strip() + if not state_path_text: + return + limit = _positive_int_env("CODE4ME_MODEL_REQUEST_LIMIT", 5) + window_seconds = _positive_float_env("CODE4ME_MODEL_REQUEST_WINDOW_SECONDS", 70.0) + _rate_limit_provider_request( + state_path=Path(state_path_text).expanduser(), + limit=limit, + window_seconds=window_seconds, + ) + + +def _rate_limit_provider_request( + *, + state_path: Path, + limit: int, + window_seconds: float, + now_fn=time, + sleep_fn=sleep, +) -> None: + state_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = state_path.with_suffix(state_path.suffix + ".lock") + with lock_path.open("a+", encoding="utf-8") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + while True: + now = float(now_fn()) + timestamps = _read_rate_limit_timestamps(state_path) + timestamps = sorted( + timestamp + for timestamp in timestamps + if now - timestamp < window_seconds + ) + if len(timestamps) < limit: + timestamps.append(now) + _write_rate_limit_timestamps(state_path, timestamps) + return + sleep_seconds = max(0.0, timestamps[0] + window_seconds - now) + logging.info( + "Model request rate limit reached: limit=%s window_seconds=%s sleeping=%.3fs", + limit, + window_seconds, + sleep_seconds, + ) + sleep_fn(sleep_seconds) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _read_rate_limit_timestamps(state_path: Path) -> list[float]: + try: + data = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(data, dict): + return [] + raw_timestamps = data.get("timestamps", []) + if not isinstance(raw_timestamps, list): + return [] + timestamps: list[float] = [] + for value in raw_timestamps: + try: + timestamps.append(float(value)) + except (TypeError, ValueError): + continue + return timestamps + + +def _write_rate_limit_timestamps(state_path: Path, timestamps: list[float]) -> None: + state_path.write_text( + json.dumps({"timestamps": timestamps}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _positive_int_env(name: str, default: int) -> int: + try: + value = int(os.getenv(name, "")) + except ValueError: + return default + return value if value > 0 else default + + +def _positive_float_env(name: str, default: float) -> float: + try: + value = float(os.getenv(name, "")) + except ValueError: + return default + return value if value > 0 else default + + +def _non_negative_int_env(name: str, default: int) -> int: + try: + value = int(os.getenv(name, "")) + except ValueError: + return default + return value if value >= 0 else default + + +def _non_negative_float_env(name: str, default: float) -> float: + try: + value = float(os.getenv(name, "")) + except ValueError: + return default + return value if value >= 0 else default + + +class ToolRegistryError(RuntimeError): + def __init__(self, message: str, *, failure_reason: str) -> None: + super().__init__(message) + self.failure_reason = failure_reason + + +class DeterministicEchoAdapter: + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + ) -> AdapterResult: + return AdapterResult( + final_response=f"Code4Me ACP echo: {prompt}", + stop_reason="end_turn", + run_status="completed", + ) + + +class ToolRegistry: + def __init__( + self, + file_tools: WorkspaceFileTools, + command_tools: WorkspaceCommandTools, + *, + event_sink: AgentEventSink | None = None, + ) -> None: + from code4me2_agent.events import ( + NoopAgentEventSink, + ThoughtEvent, + ToolCallEvent, + ) + + self._file_tools = file_tools + self._command_tools = command_tools + self._event_sink = event_sink or NoopAgentEventSink() + self._event_type = ToolCallEvent + self._thought_event_type = ThoughtEvent + + def execute( + self, tool_call: ToolCall, *, run_id: str, request_id: str + ) -> dict[str, Any]: + name = tool_call.name + arguments = dict(tool_call.arguments) + self._emit_tool_start( + tool_call, arguments, run_id=run_id, request_id=request_id + ) + try: + if name == "read_file": + result = self._file_tools.read_file( + path=str(arguments["path"]), + line_start=_optional_int(arguments.get("line_start")), + line_end=_optional_int(arguments.get("line_end")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "create_file": + result = self._file_tools.create_file( + path=str(arguments["path"]), + content=str(arguments.get("content", "")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "write_file": + result = self._file_tools.write_file( + path=str(arguments["path"]), + content=str(arguments.get("content", "")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "replace_text": + result = self._file_tools.replace_text( + path=str(arguments["path"]), + old_text=str(arguments.get("old_text", "")), + new_text=str(arguments.get("new_text", "")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "list_files": + result = self._file_tools.list_files( + path=str(arguments.get("path", ".")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "search_files": + result = self._file_tools.search_files( + query=str(arguments["query"]), + path=str(arguments.get("path", ".")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + elif name == "run_command": + result = self._command_tools.run_command( + argv=arguments["argv"], + cwd=str(arguments.get("cwd", ".")), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + else: + raise ToolRegistryError( + f"Unsupported tool name: {name}", + failure_reason="unsupported_tool", + ) + except Exception: + self._emit_tool_failed( + tool_call, arguments, run_id=run_id, request_id=request_id + ) + raise + tool_output = asdict(result) if is_dataclass(result) else dict(result) + self._emit_tool_completed( + tool_call, tool_output, run_id=run_id, request_id=request_id + ) + return tool_output + + def _emit_tool_start( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + *, + run_id: str, + request_id: str, + ) -> None: + metadata = _tool_event_metadata(tool_call.name, arguments) + if metadata is None: + return + self._event_sink.tool_call( + self._event_type( + phase="started", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id=run_id, + request_id=request_id, + title=metadata["title"], + kind=metadata["kind"], + status="pending", + path=metadata.get("path"), + raw_input=metadata.get("raw_input"), + ) + ) + + def _emit_tool_completed( + self, + tool_call: ToolCall, + tool_output: dict[str, Any], + *, + run_id: str, + request_id: str, + ) -> None: + metadata = _tool_event_metadata(tool_call.name, tool_output) + if metadata is None: + return + self._event_sink.tool_call( + self._event_type( + phase="completed", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id=run_id, + request_id=request_id, + title=metadata["title"], + kind=metadata["kind"], + status="completed", + path=metadata.get("path"), + content_text=metadata.get("content_text"), + raw_output=metadata.get("raw_output"), + ) + ) + + def _emit_tool_failed( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + *, + run_id: str, + request_id: str, + ) -> None: + metadata = _tool_event_metadata(tool_call.name, arguments) + if metadata is None: + return + self._event_sink.tool_call( + self._event_type( + phase="failed", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id=run_id, + request_id=request_id, + title=metadata["title"], + kind=metadata["kind"], + status="failed", + path=metadata.get("path"), + content_text=f"{metadata.get('content_text', metadata['title'])} failed", + ) + ) + + +class FakeOpenAICompatibleProvider: + def __init__(self, script: list[dict[str, object]]) -> None: + self._script = [dict(item) for item in script] + + def generate(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + if not self._script: + raise FakeProviderExhaustedError( + "Fake provider script exhausted before producing a final answer." + ) + current = dict(self._script.pop(0)) + delay_seconds = current.pop("delay_seconds", None) + if delay_seconds is not None: + sleep(max(0.0, float(delay_seconds))) + current["seen_messages"] = [dict(message) for message in messages] + return current + + +class OpenAICompatibleProvider: + def __init__( + self, + *, + kind: str, + base_url: str, + model: str, + api_key_env: str, + timeout_seconds: float, + auth_headers: dict[str, str] | None = None, + ) -> None: + self._kind = kind.strip() or "code4me_backend" + self._base_url = base_url.rstrip("/") + self._model = model + self._api_key_env = api_key_env + self._timeout_seconds = timeout_seconds + self._auth_headers = dict(auth_headers or {}) + + def generate(self, messages: list[dict[str, Any]]) -> ProviderTurn: + request_payload = { + "model": self._model, + "messages": [_to_openai_message(message) for message in messages], + "tools": _tool_definitions(), + "tool_choice": "auto", + } + + client = self._client() + max_429_retries = _non_negative_int_env("CODE4ME_BACKEND_429_MAX_RETRIES", 3) + retry_sleep_seconds = _non_negative_float_env("CODE4ME_BACKEND_429_RETRY_SLEEP", 10.0) + try: + for attempt in range(1, max_429_retries + 2): + logging.info( + "Agent model request via OpenAI SDK: kind=%s endpoint=%s model=%s", + self._kind, + f"{self._openai_base_url()}/chat/completions", + self._model, + ) + try: + _rate_limit_provider_request_from_env() + raw_response = client.chat.completions.with_raw_response.create(**request_payload) + logging.info( + "Agent model response via OpenAI SDK: status=%s model=%s", + raw_response.http_response.status_code, + self._model, + ) + response_payload = raw_response.http_response.json() + break + except APIStatusError as exc: + if exc.status_code == 429 and attempt <= max_429_retries: + logging.info( + "Backend model request returned HTTP 429; sleeping %.3fs before retry %s/%s", + retry_sleep_seconds, + attempt, + max_429_retries, + ) + sleep(retry_sleep_seconds) + continue + raise BackendProviderRequestError(f"The backend model request failed with HTTP {exc.status_code}: {exc.response.text}") from None + except APIError as exc: + raise BackendProviderRequestError(f"The backend model request failed: {exc}") from None + + normalized_output = _normalize_openai_provider_response(response_payload) + return ProviderTurn( + output=normalized_output, + usage=_normalize_usage( + response_payload.get("usage"), messages, normalized_output + ), + finish_reason=_response_finish_reason(response_payload), + model=str(response_payload.get("model") or self._model), + request_payload=request_payload, + raw_response=response_payload, + ) + + def _client(self) -> OpenAI: + return _OpenAIClientWithoutSdkAuth( + api_key="code4me-local-client", + base_url=self._openai_base_url(), + timeout=self._timeout_seconds, + default_headers=self._headers(), + ) + + def _headers(self) -> dict[str, str]: + headers = {"User-Agent": "code4me2-agent/0.1"} + api_key = os.getenv(self._api_key_env, "").strip() + if self._kind == "code4me_backend": + headers.update(self._auth_headers) + elif self._kind == "openai": + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + else: + raise ValueError(f"Unsupported provider kind: {self._kind}") + return headers + + def _openai_base_url(self) -> str: + if self._kind == "code4me_backend": + return f"{self._base_url}/api" + base_url = self._base_url + if not base_url.endswith("/v1"): + base_url = f"{base_url}/v1" + return base_url + +class _OpenAIClientWithoutSdkAuth(OpenAI): + @property + def auth_headers(self) -> dict[str, str]: + return {} + +class MemoryWindow: + def __init__(self, *, strategy: str, max_messages: int, max_tokens: int) -> None: + self._strategy = ( + strategy + if strategy in {"last_messages", "token_window"} + else "last_messages" + ) + self._max_messages = max(1, max_messages) + self._max_tokens = max(1, max_tokens) + self._messages: list[dict[str, Any]] = [] + + def append(self, message: dict[str, Any]) -> None: + self._messages.append(dict(message)) + + def replace_messages(self, messages: list[dict[str, Any]]) -> None: + self._messages = [dict(message) for message in messages] + + def snapshot(self) -> list[dict[str, Any]]: + return [dict(message) for message in self._messages] + + def ensure_system_message(self, content: str) -> None: + message = {"role": "system", "content": content} + if self._messages and self._messages[0].get("role") == "system": + self._messages[0] = message + return + self._messages.insert(0, message) + + def window(self) -> list[dict[str, Any]]: + pinned_system, source_messages = self._split_pinned_system() + max_messages = None if self._strategy == "token_window" else self._max_messages + + selected: list[dict[str, Any]] = [] + token_total = ( + _message_token_count(pinned_system) if pinned_system is not None else 0 + ) + for message in reversed(source_messages): + message_tokens = _message_token_count(message) + if selected and token_total + message_tokens > self._max_tokens: + break + if max_messages is not None and len(selected) >= max_messages: + break + selected.append(message) + token_total += message_tokens + selected.reverse() + if pinned_system is not None: + selected.insert(0, pinned_system) + return selected + + def _split_pinned_system( + self, + ) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]: + if self._messages and self._messages[0].get("role") == "system": + return dict(self._messages[0]), self._messages[1:] + return None, self._messages + + +class OpenAICompatibleReactAdapter: + def __init__( + self, + config: AgentConfig, + *, + telemetry: AgentTelemetryRecorder, + tool_registry: ToolRegistry, + event_sink: AgentEventSink | None = None, + ) -> None: + from code4me2_agent.events import NoopAgentEventSink, ThoughtEvent + + self._config = config + self._telemetry = telemetry + self._tool_registry = tool_registry + self._event_sink = event_sink or NoopAgentEventSink() + self._thought_event_type = ThoughtEvent + + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + ) -> AdapterResult: + provider = self._provider() + if memory is None: + memory = MemoryWindow( + strategy=self._config.adapter.memory_window.strategy, + max_messages=self._config.adapter.memory_window.max_messages, + max_tokens=self._config.adapter.memory_window.max_tokens, + ) + memory.ensure_system_message(self._system_context()) + memory.append({"role": "user", "content": prompt}) + thoughts: list[str] = [] + for iteration in range(1, self._config.adapter.max_iterations + 1): + messages = memory.window() + thought_started_at = perf_counter() + self._event_sink.thought( + self._thought_event_type( + run_id=run_id, + request_id=request_id, + text="", + phase="started", + ) + ) + self._telemetry.record( + event_type="agent.model.requested", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "iteration": iteration, + "message_count": len(messages), + "approx_token_count": sum( + _message_token_count(message) for message in messages + ), + }, + raw_payload={"messages": messages}, + ) + try: + started_at = perf_counter() + # 4 + if isinstance(provider, OpenAICompatibleProvider): + provider_turn = provider.generate(messages) + self._record_model_completed( + run_id=run_id, + request_id=request_id, + duration_ms=(perf_counter() - started_at) * 1000, + provider_turn=provider_turn, + ) + output = provider_turn.output + else: + output = provider.generate(messages) + except FakeProviderExhaustedError: + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + ) + self._record_loop_failure( + run_id=run_id, + request_id=request_id, + failure_reason="provider_exhausted", + ) + return self._failed_result( + "The fake provider stopped before returning a final answer.", + stop_reason="provider_exhausted", + ) + except BackendProviderRequestError as exc: + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + ) + self._record_loop_failure( + run_id=run_id, + request_id=request_id, + failure_reason="backend_provider_request_failed", + ) + return self._failed_result( + str(exc), + stop_reason="error", + ) + except Exception as exc: + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + ) + self._record_loop_failure( + run_id=run_id, + request_id=request_id, + failure_reason="provider_request_failed", + ) + return self._failed_result( + f"The provider request failed: {exc}", + stop_reason="error", + ) + # 5 + parsed = self._parse_output(output, run_id=run_id, request_id=request_id) + if parsed is None: + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + ) + return self._failed_result( + "The adapter could not parse the provider output.", + stop_reason="error", + ) + if parsed.thought: + thoughts.append(parsed.thought) + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + text=parsed.thought or "", + ) + if parsed.tool_calls: + memory.append( + self._assistant_tool_call_message( + parsed.tool_calls, + structured=isinstance(provider, OpenAICompatibleProvider), + ) + ) + for tool_call in parsed.tool_calls: + self._telemetry.record( + event_type="agent.tool.called", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "arguments": tool_call.arguments, + }, + raw_payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "arguments": tool_call.arguments, + }, + ) + try: + # 7 + tool_output = self._tool_registry.execute( + tool_call, + run_id=run_id, + request_id=request_id, + ) + except ToolRegistryError as exc: + return self._tool_failure_result( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason=exc.failure_reason, + error_message=str(exc), + thoughts=thoughts, + ) + except PermissionError as exc: + tool_output = { + "status": "denied", + "error": str(exc), + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } + except Exception as exc: + if tool_call.name == "run_command": + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="tool_execution_error", + error_message=str(exc), + ) + tool_output = { + "status": "failed", + "error": str(exc), + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } + memory.append( + { + "role": "tool", + "tool_call_id": tool_call.tool_call_id, + "name": tool_call.name, + "content": json.dumps(tool_output, sort_keys=True), + } + ) + continue + return self._tool_failure_result( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="tool_execution_error", + error_message=str(exc), + thoughts=thoughts, + ) + memory.append( + { + "role": "tool", + "tool_call_id": tool_call.tool_call_id, + "name": tool_call.name, + "content": json.dumps(tool_output, sort_keys=True), + } + ) + continue + if parsed.final_answer is not None: + if ( + iteration < self._config.adapter.max_iterations + and _USER_REQUESTED_FILE_CHANGE_RE.search(prompt) + and _TEXT_CONTAINS_CODE_BLOCK_RE.search(parsed.final_answer) + ): + file_path = _extract_path_from_prompt(prompt) + if file_path is None: + file_path = _extract_path_from_memory(memory) + file_content = _extract_content_from_code_block(parsed.final_answer) + if file_path is not None and file_content is not None: + tool_call = ToolCall( + tool_call_id="auto-convert", + name="write_file", + arguments={"path": file_path, "content": file_content}, + ) + try: + tool_output = self._tool_registry.execute( + tool_call, + run_id=run_id, + request_id=request_id, + ) + except ToolRegistryError: + pass + except PermissionError: + pass + else: + memory.append({"role": "assistant", "content": parsed.final_answer}) + memory.append( + self._assistant_tool_call_message( + [tool_call], + structured=isinstance(provider, OpenAICompatibleProvider), + ) + ) + memory.append({ + "role": "tool", + "tool_call_id": tool_call.tool_call_id, + "name": tool_call.name, + "content": json.dumps(tool_output, sort_keys=True), + }) + return AdapterResult( + final_response=parsed.final_answer, + stop_reason="end_turn", + run_status="completed", + thoughts=tuple(thoughts), + ) + memory.append({"role": "assistant", "content": parsed.final_answer}) + memory.append({ + "role": "user", + "content": ( + "You described the change but did not apply it. " + "Call replace_text for a focused edit, or write_file with the file path and the complete new content." + ), + }) + continue + memory.append({"role": "assistant", "content": parsed.final_answer}) + return AdapterResult( + final_response=parsed.final_answer, + stop_reason="end_turn", + run_status="completed", + thoughts=tuple(thoughts), + ) + # TODO it feels like there might be visible answer and tool call. check it out. + memory.append( + { + "role": "user", + "content": ( + "Your previous response had no visible answer and no tool call. " + "Respond with user-visible content, or call one of the available tools." + ), + } + ) + continue + + self._record_loop_failure( + run_id=run_id, + request_id=request_id, + failure_reason="max_iterations_exceeded", + ) + return self._failed_result( + "The agent reached the maximum number of iterations without a final answer.", + stop_reason="max_iterations", + thoughts=thoughts, + ) + + def _complete_thought_phase( + self, + *, + run_id: str, + request_id: str, + started_at: float, + text: str = "", + ) -> None: + self._event_sink.thought( + self._thought_event_type( + run_id=run_id, + request_id=request_id, + text=text, + phase="completed", + duration_ms=round((perf_counter() - started_at) * 1000, 3), + ) + ) + + def _failed_result( + self, + final_response: str, + *, + stop_reason: str, + thoughts: list[str] | tuple[str, ...] = (), + ) -> AdapterResult: + return AdapterResult( + final_response=final_response, + stop_reason=stop_reason, + run_status="failed", + thoughts=tuple(thoughts), + ) + + def _tool_failure_result( + self, + *, + run_id: str, + request_id: str, + tool_call: ToolCall, + failure_reason: str, + error_message: str, + thoughts: list[str] | tuple[str, ...] = (), + ) -> AdapterResult: + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason=failure_reason, + error_message=error_message, + ) + return self._failed_result( + "The adapter could not complete the requested tool call.", + stop_reason="tool_error", + thoughts=thoughts, + ) + + def _provider(self) -> FakeOpenAICompatibleProvider | OpenAICompatibleProvider: + if self._config.adapter.fake_provider.enabled: + return FakeOpenAICompatibleProvider( + self._config.adapter.fake_provider.script + ) + return OpenAICompatibleProvider( + kind=self._config.adapter.provider.kind, + base_url=self._config.adapter.provider.base_url, + model=self._config.adapter.provider.model, + api_key_env=self._config.adapter.provider.api_key_env, + timeout_seconds=self._config.adapter.provider.timeout_seconds, + auth_headers=self._config.adapter.provider.auth_headers, + ) + + def _system_context(self) -> str: + workspace_root = self._config.workspace_root.as_posix() + return ( + "You are a helpful programming assistant. " + f"The current working directory and workspace root is {workspace_root}. " + "Respond conversationally to greetings and general questions without calling tools. " + "Only use tools when the user explicitly asks you to read, search, create, edit, or list files, " + "or to run a command in the workspace. " + "When the user asks about a file's contents, always call read_file first before answering. " + "When the user asks you to change, edit, update, or modify a file's contents, " + "you MUST call replace_text for small edits or write_file with the complete new content rather than just describing the change. " + "Do not output code as text when the user expects you to apply the change. " + "When the user asks you to run, execute, compile, or build a file or command, " + "you MUST call run_command with the appropriate argv rather than just describing the command. " + "When calling tools, prefer workspace-relative paths and cwd='.' unless the user explicitly asks for an absolute path. " + "Do not invent container paths such as /workspace. " + "After file tools complete, answer directly in assistant text; do not run commands just to print a confirmation." + ) + + def _assistant_tool_call_message( + self, + tool_calls: list[ToolCall], + *, + structured: bool, + ) -> dict[str, Any]: + if structured: + return { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": tool_call.tool_call_id, + "name": tool_call.name, + "arguments": dict(tool_call.arguments), + } + for tool_call in tool_calls + ], + } + return { + "role": "assistant", + "content": json.dumps( + { + "tool_calls": [ + { + "id": tool_call.tool_call_id, + "name": tool_call.name, + "arguments": tool_call.arguments, + } + for tool_call in tool_calls + ] + }, + sort_keys=True, + ), + } + + def _parse_output( + self, + output: dict[str, Any], + *, + run_id: str, + request_id: str, + ) -> ParsedProviderOutput | None: + raw_output = output.get("output", output) + if not isinstance(raw_output, dict): + raw_output = {"text": str(raw_output)} + + tool_calls_data = raw_output.get("tool_calls") + final_answer = raw_output.get("final_answer") + if isinstance(tool_calls_data, list) or isinstance(final_answer, str): + # 6 - TODO here might be the problem in tool calling returns + fallback_tool_call = _single_json_tool_call_from_text(final_answer) + if fallback_tool_call is not None and not tool_calls_data: + return ParsedProviderOutput( + tool_calls=[fallback_tool_call], + final_answer=( + str(final_answer) + if final_answer is not None and str(final_answer).strip() + else None + ), + thought=_normalize_optional_text(raw_output.get("reasoning")), + ) + return ParsedProviderOutput( + tool_calls=_normalize_tool_calls(tool_calls_data), + final_answer=( + str(final_answer) + if final_answer is not None and str(final_answer).strip() + else None + ), + thought=_normalize_optional_text(raw_output.get("reasoning")), + ) + + text = raw_output.get("text") + if not isinstance(text, str): + self._record_parse_failure( + run_id=run_id, + request_id=request_id, + output=raw_output, + failure_reason="missing_structured_output", + ) + return None + try: + parsed_text = json.loads(text) + except json.JSONDecodeError: + self._record_parse_failure( + run_id=run_id, + request_id=request_id, + output={"text": text}, + failure_reason="invalid_json_text", + ) + return None + if not isinstance(parsed_text, dict): + self._record_parse_failure( + run_id=run_id, + request_id=request_id, + output={"text": text}, + failure_reason="json_text_not_object", + ) + return None + fallback_tool_call = _single_tool_call_from_object(parsed_text) + if fallback_tool_call is not None: + return ParsedProviderOutput( + tool_calls=[fallback_tool_call], + final_answer=None, + thought=_normalize_optional_text(raw_output.get("reasoning")), + ) + return ParsedProviderOutput( + tool_calls=_normalize_tool_calls(parsed_text.get("tool_calls")), + final_answer=( + str(parsed_text["final_answer"]) + if parsed_text.get("final_answer") is not None + else None + ), + thought=_normalize_optional_text(parsed_text.get("reasoning")), + ) + + def _record_parse_failure( + self, + *, + run_id: str, + request_id: str, + output: dict[str, Any], + failure_reason: str, + ) -> None: + self._telemetry.record( + event_type="agent.adapter.parse_failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "adapter_name": self._config.adapter.name, + "failure_reason": failure_reason, + }, + raw_payload=output, + ) + + def _record_loop_failure( + self, + *, + run_id: str, + request_id: str, + failure_reason: str, + ) -> None: + self._telemetry.record( + event_type="agent.adapter.loop_failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "adapter_name": self._config.adapter.name, + "max_iterations": self._config.adapter.max_iterations, + "failure_reason": failure_reason, + }, + ) + + def _record_tool_failure( + self, + *, + run_id: str, + request_id: str, + tool_call: ToolCall, + failure_reason: str, + error_message: str, + ) -> None: + self._telemetry.record( + event_type="agent.tool.failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "failure_reason": failure_reason, + "error_message": error_message, + }, + ) + + def _record_model_completed( + self, + *, + run_id: str, + request_id: str, + duration_ms: float, + provider_turn: ProviderTurn, + ) -> None: + usage = provider_turn.usage + self._telemetry.record( + event_type="agent.model.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "model": provider_turn.model, + "base_url": self._config.adapter.provider.base_url, + "finish_reason": provider_turn.finish_reason, + "usage": usage, + }, + metrics={ + "duration_ms": round(duration_ms, 3), + "prompt_tokens": usage.get("prompt_tokens", 0), + "completion_tokens": usage.get("completion_tokens", 0), + "total_tokens": usage.get("total_tokens", 0), + }, + raw_payload={ + "request": provider_turn.request_payload, + "response": provider_turn.raw_response, + }, + ) + + +def create_agent_adapter( + config: AgentConfig, + *, + telemetry: AgentTelemetryRecorder, + file_tools: WorkspaceFileTools, + command_tools: WorkspaceCommandTools, + event_sink: AgentEventSink | None = None, +) -> AgentAdapter: + if config.adapter.name == "openai_compatible_react": + return OpenAICompatibleReactAdapter( + config, + telemetry=telemetry, + tool_registry=ToolRegistry( + file_tools, command_tools, event_sink=event_sink + ), + event_sink=event_sink, + ) + return DeterministicEchoAdapter() + + +def _tool_event_metadata( + tool_name: str, values: dict[str, Any] +) -> dict[str, Any] | None: + if tool_name == "read_file": + path = str(values.get("path", "")).strip() + filename = path.rsplit("/", 1)[-1] or path + title = f"Read {path}" if path else "Read file" + return { + "kind": "read", + "title": title, + "path": path or None, + "content_text": f"Read file {filename}" if filename else "Read file", + "raw_input": {"path": path} if path else None, + "raw_output": {"path": path} if path else None, + } + if tool_name == "create_file": + return _edit_tool_event_metadata( + values, + title_prefix="Create", + fallback_title="Create file", + content_prefix="Created file", + ) + if tool_name == "write_file": + return _edit_tool_event_metadata( + values, + title_prefix="Update", + fallback_title="Update file", + content_prefix="Updated file", + ) + if tool_name == "replace_text": + return _edit_tool_event_metadata( + values, + title_prefix="Replace", + fallback_title="Replace text", + content_prefix="Replaced text in", + ) + return None + + +def _edit_tool_event_metadata( + values: dict[str, Any], + *, + title_prefix: str, + fallback_title: str, + content_prefix: str, +) -> dict[str, Any]: + path = str(values.get("path", "")).strip() + filename = path.rsplit("/", 1)[-1] or path + title = f"{title_prefix} {path}" if path else fallback_title + return { + "kind": "edit", + "title": title, + "path": path or None, + "content_text": f"{content_prefix} {filename}" if filename else content_prefix, + "raw_input": {"path": path} if path else None, + "raw_output": {"path": path} if path else None, + } + + +def _normalize_tool_calls(value: Any) -> list[ToolCall]: + if not isinstance(value, list): + return [] + normalized: list[ToolCall] = [] + for index, raw_tool_call in enumerate(value): + if not isinstance(raw_tool_call, dict): + continue + arguments = raw_tool_call.get("arguments", {}) + normalized.append( + ToolCall( + tool_call_id=str(raw_tool_call.get("id", f"tool-call-{index + 1}")), + name=str(raw_tool_call.get("name", "")).strip(), + arguments=dict(arguments) if isinstance(arguments, dict) else {}, + ) + ) + return [tool_call for tool_call in normalized if tool_call.name] + + +# TODO check here +def _single_json_tool_call_from_text(value: Any) -> ToolCall | None: + if not isinstance(value, str) or not value.strip(): + return None + text = value.strip() + result = _try_parse_json(text) + if result is not None: + return result + for match in _CODE_FENCE_RE.finditer(text): + result = _try_parse_json(match.group(1).strip()) + if result is not None: + return result + idx = text.find('{"name":') + if idx >= 0: + brace_count = 0 + for i in range(idx, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + result = _try_parse_json(text[idx : i + 1]) + if result is not None: + return result + return None + + +_CODE_FENCE_RE = re.compile(r"```(?:json)?\s*\n?(.*?)```", re.DOTALL) +_USER_REQUESTED_FILE_CHANGE_RE = re.compile( + r"(?:change|edit|update|modify|write|replace|fix|save)\s", + re.IGNORECASE, +) +_TEXT_CONTAINS_CODE_BLOCK_RE = re.compile(r"```[\w]*\n", re.MULTILINE) +_CODE_BLOCK_CONTENT_RE = re.compile(r"```(?:\w+)?\n(.*?)```", re.DOTALL) +_PATH_FROM_PROMPT_RE = re.compile( + r"(?:change|edit|update|modify|write|replace|fix|save)\s+" + r"(?:the\s+|file\s+)?(\S+\.\w+)", + re.IGNORECASE, +) + + +def _extract_path_from_prompt(prompt: str) -> str | None: + match = _PATH_FROM_PROMPT_RE.search(prompt) + if match is None: + return None + return match.group(1).strip() + + +def _extract_path_from_memory(memory: MemoryWindow) -> str | None: + for message in reversed(memory._messages): + if message.get("role") == "tool" and message.get("name") == "read_file": + content = message.get("content", "") + if isinstance(content, str): + try: + data = json.loads(content) + if isinstance(data, dict): + path = data.get("path") + if isinstance(path, str) and path.strip(): + return path.strip() + except (json.JSONDecodeError, TypeError): + pass + return None + + +def _extract_content_from_code_block(text: str) -> str | None: + match = _CODE_BLOCK_CONTENT_RE.search(text) + if match is None: + return None + return match.group(1).strip() + + +def _try_parse_json(text: str) -> ToolCall | None: + if not text: + return None + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + return _single_tool_call_from_object(parsed) + + +def _single_tool_call_from_object(value: dict[str, Any]) -> ToolCall | None: + name = str(value.get("name", "")).strip() + if name not in _known_tool_names(): + return None + arguments = value.get("arguments", {}) + if not isinstance(arguments, dict): + return None + return ToolCall( + tool_call_id=str(value.get("id", "tool-call-1")), + name=name, + arguments=dict(arguments), + ) + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _known_tool_names() -> set[str]: + names: set[str] = set() + for tool in _tool_definitions(): + if not isinstance(tool, dict): + continue + function = tool.get("function") + if isinstance(function, dict): + name = str(function.get("name", "")).strip() + if name: + names.add(name) + return names + + +def _normalize_optional_text(value: Any) -> str | None: + if not isinstance(value, str): + return None + normalized = value.strip() + return normalized or None + + +def _message_token_count(message: dict[str, Any]) -> int: + parts: list[str] = [] + for key in ("role", "content", "tool_call_id", "name"): + value = message.get(key) + if value: + parts.append(str(value)) + tool_calls = message.get("tool_calls") + if tool_calls: + parts.append(json.dumps(tool_calls, sort_keys=True)) + text = " ".join(parts).strip() + if not text: + return 1 + return max(1, len(text.split())) + + +def _tool_definitions() -> list[dict[str, Any]]: + return [ + _function_tool( + "read_file", + "Read a UTF-8 text file inside the configured workspace. Optionally pass 1-based line_start and line_end.", + { + "path": {"type": "string"}, + "line_start": {"type": "integer", "minimum": 1}, + "line_end": {"type": "integer", "minimum": 1}, + }, + ["path"], + ), + _function_tool( + "create_file", + ( + "Create a new UTF-8 text file inside the configured workspace. " + "Use this only when the target file does not already exist. " + "Do not call write_file immediately after a successful create_file for the same path " + "unless you are intentionally changing the content again." + ), + { + "path": {"type": "string"}, + "content": {"type": "string"}, + }, + ["path", "content"], + ), + _function_tool( + "write_file", + ( + "Overwrite a UTF-8 text file inside the configured workspace. " + "Use this when the file already exists or when you intentionally need to replace its content. " + "If you just created the file with create_file and the content is already correct, do not call write_file again." + ), + { + "path": {"type": "string"}, + "content": {"type": "string"}, + }, + ["path", "content"], + ), + _function_tool( + "replace_text", + ( + "Replace the first exact occurrence of old_text in a UTF-8 text file. " + "Prefer this for small edits instead of sending an entire file through write_file. " + "Call read_file first so old_text exactly matches the current file content." + ), + { + "path": {"type": "string"}, + "old_text": {"type": "string"}, + "new_text": {"type": "string"}, + }, + ["path", "old_text", "new_text"], + ), + _function_tool( + "list_files", + "List files underneath a workspace-relative path.", + { + "path": {"type": "string"}, + }, + [], + ), + _function_tool( + "search_files", + "Search for text matches in workspace files.", + { + "query": {"type": "string"}, + "path": {"type": "string"}, + }, + ["query"], + ), + _function_tool( + "run_command", + "Run an allowlisted workspace command without a shell.", + { + "argv": { + "type": "array", + "items": {"type": "string"}, + }, + "cwd": {"type": "string"}, + }, + ["argv"], + ), + ] + + +def _function_tool( + name: str, + description: str, + properties: dict[str, Any], + required: list[str], +) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": properties, + "required": required, + "additionalProperties": False, + }, + }, + } + + +def _to_openai_message(message: dict[str, Any]) -> dict[str, Any]: + role = str(message.get("role", "user")) + if role == "assistant" and message.get("tool_calls"): + return { + "role": "assistant", + "content": message.get("content", ""), + "tool_calls": [ + { + "id": str(tool_call.get("id", "")), + "type": "function", + "function": { + "name": str(tool_call.get("name", "")), + "arguments": json.dumps( + tool_call.get("arguments", {}), sort_keys=True + ), + }, + } + for tool_call in message.get("tool_calls", []) + if isinstance(tool_call, dict) + ], + } + if role == "tool": + return { + "role": "tool", + "tool_call_id": str(message.get("tool_call_id", "")), + "content": str(message.get("content", "")), + } + return { + "role": role, + "content": str(message.get("content", "")), + } + + +def _normalize_openai_provider_response( + response_payload: dict[str, Any] +) -> dict[str, Any]: + choices = response_payload.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("Provider response did not include choices.") + first_choice = choices[0] if isinstance(choices[0], dict) else {} + message = first_choice.get("message", {}) + if not isinstance(message, dict): + raise RuntimeError("Provider response choice message was not an object.") + + tool_calls = [] + raw_tool_calls = message.get("tool_calls", []) + if isinstance(raw_tool_calls, list): + for index, raw_tool_call in enumerate(raw_tool_calls): + if not isinstance(raw_tool_call, dict): + continue + function_data = raw_tool_call.get("function", {}) + if not isinstance(function_data, dict): + continue + raw_arguments = function_data.get("arguments", "{}") + if not isinstance(raw_arguments, str): + raw_arguments = json.dumps(raw_arguments) + try: + arguments = json.loads(raw_arguments) + except json.JSONDecodeError: + arguments = {} + if not isinstance(arguments, dict): + arguments = {} + name = str(function_data.get("name", "")).strip() + if not name: + continue + tool_calls.append( + { + "id": str(raw_tool_call.get("id", f"tool-call-{index + 1}")), + "name": name, + "arguments": arguments, + } + ) + + final_answer = message.get("content") + if isinstance(final_answer, list): + final_answer = "".join( + str(block.get("text", "")) + for block in final_answer + if isinstance(block, dict) and block.get("type") == "text" + ) + + normalized: dict[str, Any] = {"tool_calls": tool_calls} + reasoning = _normalize_optional_text(message.get("reasoning")) + if reasoning is not None: + normalized["reasoning"] = reasoning + if final_answer is not None: + normalized["final_answer"] = str(final_answer) + return normalized + + +def _response_finish_reason(response_payload: dict[str, Any]) -> str | None: + choices = response_payload.get("choices") + if not isinstance(choices, list) or not choices: + return None + first_choice = choices[0] if isinstance(choices[0], dict) else {} + finish_reason = first_choice.get("finish_reason") + return str(finish_reason) if finish_reason is not None else None + + +def _normalize_usage( + raw_usage: Any, + messages: list[dict[str, Any]], + output: dict[str, Any], +) -> dict[str, int]: + if isinstance(raw_usage, dict): + prompt_tokens = _int_or_zero(raw_usage.get("prompt_tokens")) + completion_tokens = _int_or_zero(raw_usage.get("completion_tokens")) + total_tokens = _int_or_zero(raw_usage.get("total_tokens")) + if total_tokens <= 0: + total_tokens = prompt_tokens + completion_tokens + if prompt_tokens > 0 or completion_tokens > 0 or total_tokens > 0: + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + prompt_tokens = sum(_message_token_count(message) for message in messages) + completion_tokens = _estimated_output_token_count(output) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def _estimated_output_token_count(output: dict[str, Any]) -> int: + parts: list[str] = [] + final_answer = output.get("final_answer") + if final_answer: + parts.append(str(final_answer)) + tool_calls = output.get("tool_calls", []) + if isinstance(tool_calls, list): + parts.append(json.dumps(tool_calls, sort_keys=True)) + text = " ".join(part for part in parts if part).strip() + if not text: + return 1 + return max(1, len(text.split())) + + +def _int_or_zero(value: Any) -> int: + try: + return max(0, int(value)) + except (TypeError, ValueError): + return 0 diff --git a/src/code4me2_agent/async_bridge.py b/src/code4me2_agent/async_bridge.py new file mode 100644 index 0000000..181a481 --- /dev/null +++ b/src/code4me2_agent/async_bridge.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +from queue import Queue +from threading import Thread +from typing import Any + + +def run_awaitable_blocking(awaitable: Any, *, timeout_seconds: float | None = None) -> object: + wrapped = awaitable if timeout_seconds is None else asyncio.wait_for(awaitable, timeout_seconds) + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(wrapped) + return _run_awaitable_in_worker_thread(wrapped) + + +def _run_awaitable_in_worker_thread(awaitable: Any) -> object: + outcome: Queue[tuple[bool, object]] = Queue(maxsize=1) + + def worker() -> None: + try: + outcome.put((True, asyncio.run(awaitable))) + except BaseException as exc: # noqa: BLE001 + outcome.put((False, exc)) + + thread = Thread(target=worker, daemon=True) + thread.start() + thread.join() + succeeded, value = outcome.get() + if succeeded: + return value + raise value + + +class EventLoopAsyncRunner: + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + + def run(self, awaitable: Any, *, timeout_seconds: float | None = None) -> object: + if self._loop.is_closed() or not self._loop.is_running(): + return run_awaitable_blocking(awaitable, timeout_seconds=timeout_seconds) + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if running_loop is self._loop: + raise RuntimeError("Cannot block on an ACP client call from the ACP event loop.") + wrapped = awaitable if timeout_seconds is None else asyncio.wait_for(awaitable, timeout_seconds) + future = asyncio.run_coroutine_threadsafe(wrapped, self._loop) + try: + return future.result(timeout=timeout_seconds) + except concurrent.futures.TimeoutError as exc: + future.cancel() + raise TimeoutError("Timed out waiting for ACP client response.") from exc diff --git a/src/code4me2_agent/cli.py b/src/code4me2_agent/cli.py new file mode 100644 index 0000000..eebdc70 --- /dev/null +++ b/src/code4me2_agent/cli.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import shutil +import sys +from pathlib import Path + +from code4me2_agent.acp_runtime import run_acp_stdio +from code4me2_agent.config import AgentConfig + + +DEFAULT_CONFIG_PATH = os.path.expanduser("~/.code4me/agent-config.json") +DEFAULT_ACP_JSON_PATH = os.path.expanduser("~/.code4me/acp.json") + +# Scaffold written by `code4me2-agent init`. Deliberately specifies no model, +# provider or API key: those come from the agent profile the backend assigns +# (GET /api/acp/agent-config), so hardcoding them here would let a stale local +# file silently override a study assignment. +# +# `max_iterations` and the command allowlist are also server-overridable — the +# values here only apply before the first successful authentication. +DEFAULT_AGENT_CONFIG: dict[str, object] = { + "adapter": { + "name": "openai_compatible_react", + "max_iterations": 10, + "provider": { + # Route model calls through the backend by default. The assigned + # profile switches this to a direct provider when it names a + # base_url of its own. + "kind": "code4me_backend", + # base_url / model / api_key_env intentionally omitted — supplied + # by the backend's agent-config after authentication. + }, + "memory_window": { + "scope": "session", + "strategy": "token_window", + "max_messages": 20, + "max_tokens": 6000, + }, + }, + "commands": { + # Starting allowlist for local command execution, narrowed further by + # the server's agent-config when one is configured. Write-capable + # entries are included because the agent's purpose is editing code; the + # workspace confinement in file_tools is what bounds the damage. + "allowlist": [ + "pwd", "ls", "grep", "cat", "git", "rg", "echo", + "bash", "sh", "mkdir", "rm", "mv", "cp", "chmod", + "which", "head", "tail", "sort", "wc", "diff", "find", "make", + ], + }, + "telemetry": { + "trace_path": ".code4me/acp-trace.jsonl", + # Local JSONL trace detail. Note this does NOT control what the server + # stores: content persistence is decided server-side from the user's + # store_agent_content preference, regardless of this flag. + "raw_capture_enabled": True, + }, +} + +DEFAULT_ACP_JSON: dict[str, object] = { + "default_mcp_settings": { + "use_idea_mcp": True, + "use_custom_mcp": True, + }, + "agent_servers": { + "Code4Me Agent": { + "command": "code4me2-agent", + "args": [], + }, + }, +} + + +def _resolve_agent_command() -> str: + current_command = Path(sys.argv[0]).name + if current_command: + resolved_command = shutil.which(current_command) + if resolved_command: + return resolved_command + + for command in ("code4me2-agent"): + resolved_command = shutil.which(command) + if resolved_command: + return resolved_command + + return sys.argv[0] or "code4me2-agent" + + +def _build_acp_json() -> dict[str, object]: + acp_json = json.loads(json.dumps(DEFAULT_ACP_JSON)) + acp_json["agent_servers"]["Code4Me Agent"]["command"] = _resolve_agent_command() + return acp_json + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the Code4Me ACP agent over stdio.") + parser.add_argument( + "--config", + default=DEFAULT_CONFIG_PATH, + help="Path to agent config JSON file (default: ~/.code4me/agent-config.json).", + ) + parser.add_argument( + "--setup", + action="store_true", + help="Install default config files to ~/.code4me/ and print setup instructions.", + ) + return parser + + +def configure_logging() -> None: + level_name = os.environ.get("CODE4ME_AGENT_LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + +def _do_setup() -> None: + config_dir = Path("~/.code4me").expanduser() + config_dir.mkdir(parents=True, exist_ok=True) + + agent_config_path = config_dir / "agent-config.json" + if not agent_config_path.exists(): + agent_config_path.write_text(json.dumps(DEFAULT_AGENT_CONFIG, indent=2) + "\n") + print(f"Created {agent_config_path}") + else: + print(f"Already exists: {agent_config_path} (skipped)") + + acp_json_path = DEFAULT_ACP_JSON_PATH + if not os.path.exists(acp_json_path): + with open(acp_json_path, "w") as f: + json.dump(_build_acp_json(), f, indent=2) + f.write("\n") + print(f"Created {acp_json_path}") + else: + print(f"Already exists: {acp_json_path} (skipped)") + + print() + print("Setup complete. To use the Code4Me agent in JetBrains AI Assistant:") + print(" 1. Make sure JetBrains AI Assistant points to the acp.json above") + print(" (Settings → Tools → AI Assistant → Custom Agent Server)") + print(" 2. Restart IntelliJ") + print(" 3. Run: docker compose -f docker-compose.dev-arm.yaml up") + print(" 4. Run: ollama serve") + print(" 5. Run: ./gradlew runIde") + + +def main(argv: list[str] | None = None) -> None: + configure_logging() + args = build_parser().parse_args(argv) + + if args.setup: + _do_setup() + sys.exit(0) + + config = AgentConfig.from_file(args.config) + asyncio.run(run_acp_stdio(config)) + + +if __name__ == "__main__": + main() diff --git a/src/code4me2_agent/command_tools.py b/src/code4me2_agent/command_tools.py new file mode 100644 index 0000000..ea1a73a --- /dev/null +++ b/src/code4me2_agent/command_tools.py @@ -0,0 +1,692 @@ +from __future__ import annotations + +import asyncio +import inspect +import subprocess +from dataclasses import dataclass +from pathlib import Path +from time import perf_counter +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import run_awaitable_blocking +from code4me2_agent.telemetry import AgentTelemetryRecorder + +if TYPE_CHECKING: + from code4me2_agent.config import AgentConfig + + +@dataclass(frozen=True) +class CommandResult: + argv: list[str] + cwd: str + backend_type: str + exit_code: int | None + stdout: str + stderr: str + timed_out: bool + output_truncated: bool + + +@dataclass(frozen=True) +class AcpCommandExecution: + exit_code: int | None + stdout: str + stderr: str + timed_out: bool + output_truncated: bool + + +class AcpCommandBackend: + def __init__(self, *, client: object, session_id: str, async_runner: object | None = None) -> None: + self._client = client + self._session_id = session_id + self._async_runner = async_runner + + @property + def terminal_enabled(self) -> bool: + return True + + def run_command( + self, + *, + command: str, + args: list[str], + cwd: str, + max_output_bytes: int, + timeout_seconds: float, + ) -> AcpCommandExecution: + terminal_id = self._create_terminal( + command=command, + args=args, + cwd=cwd, + output_byte_limit=max_output_bytes, + ) + + timed_out = False + wait_response: object | None = None + try: + try: + wait_response = self._wait_for_exit(terminal_id=terminal_id, timeout_seconds=timeout_seconds) + except TimeoutError: + timed_out = True + self._kill_terminal(terminal_id=terminal_id) + + output_response = self._terminal_output(terminal_id=terminal_id) + stdout, stderr, output_truncated, output_exit_code = self._parse_output_response(output_response) + exit_code = self._parse_exit_code(wait_response) + if exit_code is None: + exit_code = output_exit_code + return AcpCommandExecution( + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + ) + finally: + self._release_terminal(terminal_id=terminal_id) + + def _create_terminal( + self, + *, + command: str, + args: list[str], + cwd: str, + output_byte_limit: int, + ) -> str: + method = _resolve_method(self._client, ["create_terminal", "terminal_create"]) + response = self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "command": command, + "args": args, + "cwd": cwd, + "output_byte_limit": output_byte_limit, + }, + camel_kwargs={ + "sessionId": self._session_id, + "command": command, + "args": args, + "cwd": cwd, + "outputByteLimit": output_byte_limit, + }, + ) + ) + if isinstance(response, dict): + terminal_id = response.get("terminal_id") or response.get("terminalId") + else: + terminal_id = getattr(response, "terminal_id", None) or getattr(response, "terminalId", None) + if terminal_id is None: + raise RuntimeError("ACP terminal backend did not return a terminal ID.") + return str(terminal_id) + + def _wait_for_exit(self, *, terminal_id: str, timeout_seconds: float) -> object: + method = _resolve_method( + self._client, + ["wait_for_terminal_exit", "terminal_wait_for_exit"], + ) + return self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ), + timeout_seconds=timeout_seconds, + ) + + def _terminal_output(self, *, terminal_id: str) -> object: + method = _resolve_method( + self._client, + ["get_terminal_output", "terminal_output"], + ) + return self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _kill_terminal(self, *, terminal_id: str) -> None: + method = _resolve_method(self._client, ["kill_terminal", "terminal_kill"]) + self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _release_terminal(self, *, terminal_id: str) -> None: + method = _resolve_method(self._client, ["release_terminal", "terminal_release"]) + self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _run_client_call(self, result: object, timeout_seconds: float | None = None) -> object: + if not inspect.isawaitable(result): + return result + if self._async_runner is not None: + return self._async_runner.run(result, timeout_seconds=timeout_seconds) + return run_awaitable_blocking(result, timeout_seconds=timeout_seconds) + + def _parse_output_response(self, response: object) -> tuple[str, str, bool, int | None]: + if isinstance(response, dict): + output = str(response.get("output", "")) + stderr = str(response.get("stderr", "")) + truncated = bool(response.get("truncated", False)) + exit_status = response.get("exit_status") or response.get("exitStatus") or {} + else: + output = str(getattr(response, "output", "")) + stderr = str(getattr(response, "stderr", "")) + truncated = bool(getattr(response, "truncated", False)) + exit_status = getattr(response, "exit_status", None) or getattr(response, "exitStatus", None) or {} + exit_code = _extract_exit_code(exit_status) + return output, stderr, truncated, exit_code + + def _parse_exit_code(self, response: object | None) -> int | None: + if response is None: + return None + return _extract_exit_code(response) + + +def build_acp_command_backend( + *, + client: object, + session_id: str, + client_capabilities: object | None, + async_runner: object | None = None, +) -> AcpCommandBackend | None: + terminal_capability = capability_value(client_capabilities, "terminal") + if not bool(terminal_capability): + return None + + required_method_groups = [ + ["create_terminal", "terminal_create"], + ["wait_for_terminal_exit", "terminal_wait_for_exit"], + ["get_terminal_output", "terminal_output"], + ["release_terminal", "terminal_release"], + ["kill_terminal", "terminal_kill"], + ] + for names in required_method_groups: + if _resolve_method(client, names) is None: + return None + + return AcpCommandBackend(client=client, session_id=session_id, async_runner=async_runner) + + +class WorkspaceCommandTools: + def __init__( + self, + config: AgentConfig, + *, + acp_backend: object | None = None, + telemetry: AgentTelemetryRecorder | None = None, + allowlisted_commands: set[str] | None = None, + timeout_seconds: float | None = None, + max_output_bytes: int | None = None, + ) -> None: + self._config = config + self._acp_backend = acp_backend + self._telemetry = telemetry or AgentTelemetryRecorder(config) + command_config = config.commands + self._allowlisted_commands = set(allowlisted_commands or command_config.allowlisted_commands) + selected_timeout_seconds = command_config.timeout_seconds if timeout_seconds is None else timeout_seconds + selected_max_output_bytes = command_config.max_output_bytes if max_output_bytes is None else max_output_bytes + self._timeout_seconds = float(selected_timeout_seconds) + self._max_output_bytes = max(1, int(selected_max_output_bytes)) + + def run_command( + self, + argv: object, + *, + cwd: str = ".", + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> CommandResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + + normalized_argv = self._normalize_argv_or_record_denial( + argv=argv, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + command_name = self._command_name_or_record_denial( + argv=normalized_argv, + cwd=cwd, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + self._validate_allowlist_or_record_denial( + command_name=command_name, + argv=normalized_argv, + cwd=cwd, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + resolved_cwd = self._resolve_cwd_or_record_denial( + cwd=cwd, + argv=normalized_argv, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + + relative_cwd = "." if resolved_cwd == self._config.workspace_root else self._relative_path(resolved_cwd) + acp_run_command = self._acp_run_command() + if acp_run_command is not None: + backend_type = "acp" + acp_result = acp_run_command( + command=command_name, + args=normalized_argv[1:], + cwd=str(resolved_cwd), + max_output_bytes=self._max_output_bytes, + timeout_seconds=self._timeout_seconds, + ) + stdout_raw = acp_result.stdout + stderr_raw = acp_result.stderr + exit_code = acp_result.exit_code + timed_out = acp_result.timed_out + output_truncated = acp_result.output_truncated + else: + backend_type = "local" + stdout_raw, stderr_raw, exit_code, timed_out = self._run_local( + argv=normalized_argv, + cwd=resolved_cwd, + ) + output_truncated = False + + stdout, stdout_truncated = _truncate_to_max_bytes(stdout_raw, self._max_output_bytes) + stderr, stderr_truncated = _truncate_to_max_bytes(stderr_raw, self._max_output_bytes) + output_truncated = output_truncated or stdout_truncated or stderr_truncated + status = "timeout" if timed_out else "completed" + + self._record_tool_event( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + argv=normalized_argv, + cwd=relative_cwd, + status=status, + backend_type=backend_type, + started_at=started_at, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + extra_payload={ + "command": command_name, + "stdout_bytes": len(stdout.encode("utf-8")), + "stderr_bytes": len(stderr.encode("utf-8")), + "max_output_bytes": self._max_output_bytes, + }, + ) + + return CommandResult( + argv=normalized_argv, + cwd=relative_cwd, + backend_type=backend_type, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + ) + + def _run_local( + self, + *, + argv: list[str], + cwd: Path, + ) -> tuple[str, str, int | None, bool]: + try: + completed = subprocess.run( + argv, + cwd=str(cwd), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=self._timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + return ( + _coerce_output_text(exc.stdout), + _coerce_output_text(exc.stderr), + None, + True, + ) + + return ( + completed.stdout or "", + completed.stderr or "", + completed.returncode, + False, + ) + + def _acp_run_command(self) -> Any | None: + if not bool(getattr(self._acp_backend, "terminal_enabled", False)): + return None + operation = getattr(self._acp_backend, "run_command", None) + if callable(operation): + return operation + return None + + def _command_name_or_record_denial( + self, + *, + argv: list[str], + cwd: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> str: + command_name = Path(argv[0]).name + if command_name != argv[0]: + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=cwd, + denial_reason="command_path_not_allowed", + ) + raise PermissionError("Command argv[0] must be an allowlisted command name, not a path.") + return command_name + + def _normalize_argv_or_record_denial( + self, + *, + argv: object, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> list[str]: + if isinstance(argv, str): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[argv], + cwd=".", + denial_reason="argv_must_be_array", + ) + raise TypeError("Command execution requires argv as a list/tuple, not a shell string.") + + if not isinstance(argv, (list, tuple)): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[], + cwd=".", + denial_reason="invalid_argv", + ) + raise TypeError("argv must be a list or tuple of strings.") + + normalized: list[str] = [] + for arg in argv: + if not isinstance(arg, str): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[str(item) for item in argv], + cwd=".", + denial_reason="invalid_argv", + ) + raise TypeError("argv must contain only strings.") + normalized.append(arg) + + if not normalized or not normalized[0].strip(): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=normalized, + cwd=".", + denial_reason="invalid_argv", + ) + raise ValueError("argv must contain a non-empty command at index 0.") + + return normalized + + def _validate_allowlist_or_record_denial( + self, + *, + command_name: str, + argv: list[str], + cwd: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> None: + if command_name in self._allowlisted_commands: + return + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=cwd, + denial_reason="command_not_allowlisted", + ) + raise PermissionError(f"Command is not in the configured allowlist: {command_name}") + + def _resolve_cwd_or_record_denial( + self, + *, + cwd: str, + argv: list[str], + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> Path: + try: + return self._resolve_workspace_path(cwd) + except PermissionError: + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=cwd, + denial_reason="outside_workspace_root", + ) + raise + + def _resolve_workspace_path(self, path: str) -> Path: + workspace_root = self._config.workspace_root.resolve() + requested_path = Path(path).expanduser() + if not requested_path.is_absolute(): + requested_path = workspace_root / requested_path + resolved_path = requested_path.resolve() + if resolved_path != workspace_root and workspace_root not in resolved_path.parents: + raise PermissionError("Path is outside the configured workspace root.") + return resolved_path + + def _relative_path(self, path: Path) -> str: + return path.relative_to(self._config.workspace_root).as_posix() + + def _record_denial( + self, + *, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + argv: list[str], + cwd: str, + denial_reason: str, + ) -> None: + self._record_tool_event( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + argv=argv, + cwd=cwd, + status="denied", + backend_type="local", + started_at=started_at, + exit_code=None, + stdout="", + stderr="", + timed_out=False, + output_truncated=False, + denial_reason=denial_reason, + ) + + def _record_tool_event( + self, + *, + tool_call_id: str, + run_id: str, + request_id: str, + argv: list[str], + cwd: str, + status: str, + backend_type: str, + started_at: float, + exit_code: int | None, + stdout: str, + stderr: str, + timed_out: bool, + output_truncated: bool, + denial_reason: str | None = None, + extra_payload: dict[str, object] | None = None, + ) -> None: + event_type = "agent.tool.denied" if status == "denied" else "agent.tool.completed" + payload: dict[str, object] = { + "tool_name": "run_command", + "tool_call_id": tool_call_id, + "backend_type": backend_type, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "status": status, + "argv": argv, + "cwd": cwd, + "exit_code": exit_code, + "timed_out": timed_out, + "output_truncated": output_truncated, + "stdout": stdout, + "stderr": stderr, + } + payload.update(extra_payload or {}) + if denial_reason is not None: + payload["denial_reason"] = denial_reason + + self._telemetry.record( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload=payload, + raw_payload=None, + ) +def _resolve_method(client: object, names: list[str]) -> Any | None: + for name in names: + method = getattr(client, name, None) + if callable(method): + return method + return None + + +def _invoke_with_name_fallback(method: Any, *, snake_kwargs: dict[str, object], camel_kwargs: dict[str, object]) -> object: + try: + return method(**snake_kwargs) + except TypeError: + return method(**camel_kwargs) + + +def _extract_exit_code(response: object) -> int | None: + if isinstance(response, dict): + raw_exit_code = response.get("exit_code") + if raw_exit_code is None: + raw_exit_code = response.get("exitCode") + else: + raw_exit_code = getattr(response, "exit_code", None) + if raw_exit_code is None: + raw_exit_code = getattr(response, "exitCode", None) + if raw_exit_code is None: + return None + try: + return int(raw_exit_code) + except (TypeError, ValueError): + return None + + +def _truncate_to_max_bytes(value: str, max_output_bytes: int) -> tuple[str, bool]: + raw_bytes = value.encode("utf-8") + if len(raw_bytes) <= max_output_bytes: + return value, False + + truncated_bytes = raw_bytes[-max_output_bytes:] + while truncated_bytes: + try: + return truncated_bytes.decode("utf-8"), True + except UnicodeDecodeError: + truncated_bytes = truncated_bytes[1:] + return "", True + + +def _coerce_output_text(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value diff --git a/src/code4me2_agent/config.py b/src/code4me2_agent/config.py new file mode 100644 index 0000000..83b42f2 --- /dev/null +++ b/src/code4me2_agent/config.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from uuid import uuid4 + + +@dataclass(frozen=True) +class UploadConfig: + enabled: bool = False + ingest_url: str | None = None + batch_size: int = 50 + timeout_seconds: float = 5.0 + auth_headers: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CommandConfig: + allowlisted_commands: list[str] = field(default_factory=list) + timeout_seconds: float = 10.0 + max_output_bytes: int = 16384 + + +@dataclass(frozen=True) +class MemoryWindowConfig: + scope: str = "prompt" + strategy: str = "last_messages" + max_messages: int = 12 + max_tokens: int = 16000 + + +@dataclass(frozen=True) +class FakeProviderConfig: + enabled: bool = False + script: list[dict[str, object]] = field(default_factory=list) + + +@dataclass(frozen=True) +class OpenAICompatibleProviderConfig: + """Where this agent sends its model calls. + + There is deliberately **no hardcoded model or provider default here.** The + backend is authoritative: after ACP authentication the runtime fetches + ``GET /api/acp/agent-config`` and overwrites these fields from the agent + profile the user is assigned to (see ``AgentConfig.with_server_overrides``). + Baking in a local Ollama default — as the original did — meant a study + assignment could be silently ignored whenever the local config disagreed, + which is exactly the failure the A/B design can't tolerate. + + ``kind`` selects how the call is routed: + + * ``code4me_backend`` — through the backend's own OpenAI-compatible bridge. + Used when the assigned profile leaves ``base_url`` unset, so the backend + picks the upstream and the call is observable server-side too. + * ``openai`` — straight to the ``base_url`` the profile names. Any + OpenAI-compatible endpoint qualifies (Ollama, Groq, OpenRouter, OpenAI); + per merge decision 6 the wire format is the only requirement. + + ``api_key_env`` is the *name* of the environment variable holding the + credential, never the credential itself: the backend sends only the name + (``api_key_ref``) and the key is resolved here, in the agent's own process. + """ + + kind: str = "code4me_backend" + # Empty = not yet configured. Filled from the server's agent-config; for + # `code4me_backend` this is the backend's own root URL. + base_url: str = "" + model: str = "" + api_key_env: str = "" + timeout_seconds: float = 90.0 + auth_headers: dict[str, str] = field(default_factory=dict) + + @property + def is_configured(self) -> bool: + """Whether this provider has enough to make a call.""" + return bool(self.base_url.strip() and self.model.strip()) + + +@dataclass(frozen=True) +class AdapterConfig: + name: str = "deterministic_echo" + max_iterations: int = 8 + memory_window: MemoryWindowConfig = field(default_factory=MemoryWindowConfig) + fake_provider: FakeProviderConfig = field(default_factory=FakeProviderConfig) + provider: OpenAICompatibleProviderConfig = field( + default_factory=OpenAICompatibleProviderConfig + ) + + +@dataclass(frozen=True) +class ServerAgentConfig: + """The runtime configuration the backend hands down. + + Mirrors the ``GET /api/acp/agent-config`` response, which derives every + field from the agent profile the user is assigned to. All fields are + optional so a partial response (or an older backend) leaves the + corresponding local value untouched rather than blanking it. + """ + + agent_profile: str | None = None + framework_version: str | None = None + model: str | None = None + base_url: str | None = None + api_key_ref: str | None = None + commands_allowlist: list[str] | None = None + tools: list[str] | None = None + max_iterations: int | None = None + max_context_tokens: int | None = None + approval_policy: str | None = None + temperature: float | None = None + # Advisory: the server enforces content storage itself. Used only to avoid + # transmitting content that would be discarded anyway — never to enable + # capture, which the agent has no power to do. + store_agent_content: bool = True + + @classmethod + def from_payload(cls, payload: dict) -> "ServerAgentConfig": + """Parse an agent-config response, ignoring anything malformed. + + Every field is validated independently: one bad value costs that + override, not the whole config, because a partially-usable server + config still beats falling back to a local guess. + """ + + def _clean_str(key: str) -> str | None: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def _clean_str_list(key: str) -> list[str] | None: + value = payload.get(key) + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return None + + def _clean_positive_int(key: str) -> int | None: + value = payload.get(key) + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 1: + return value + return None + + temperature = payload.get("temperature") + if not isinstance(temperature, (int, float)) or isinstance(temperature, bool): + temperature = None + + return cls( + agent_profile=_clean_str("agent_profile"), + framework_version=_clean_str("framework_version"), + model=_clean_str("model"), + base_url=_clean_str("base_url"), + api_key_ref=_clean_str("api_key_ref"), + commands_allowlist=_clean_str_list("commands_allowlist"), + tools=_clean_str_list("tools"), + max_iterations=_clean_positive_int("max_iterations"), + max_context_tokens=_clean_positive_int("max_context_tokens"), + approval_policy=_clean_str("approval_policy"), + temperature=float(temperature) if temperature is not None else None, + store_agent_content=bool(payload.get("store_agent_content", True)), + ) + + @property + def has_overrides(self) -> bool: + return any( + value is not None + for value in ( + self.model, + self.base_url, + self.api_key_ref, + self.commands_allowlist, + self.tools, + self.max_iterations, + self.max_context_tokens, + ) + ) + + +@dataclass(frozen=True) +class AgentConfig: + workspace_root: Path + trace_path: Path + session_id: str + raw_capture_enabled: bool = False + upload: UploadConfig = field(default_factory=UploadConfig) + commands: CommandConfig = field(default_factory=CommandConfig) + adapter: AdapterConfig = field(default_factory=AdapterConfig) + + def with_server_overrides( + self, server: ServerAgentConfig, *, backend_url: str | None = None + ) -> "AgentConfig": + """Return a copy with the backend's assigned configuration applied. + + This is where the assigned agent profile actually takes hold of the + runtime. Precedence is deliberate: **server wins**, and the local + config file is only a fallback for anything the server didn't specify. + + Routing follows from whether the profile named a ``base_url``: + + * it did → talk to that endpoint directly (``kind="openai"``), which is + what makes "any OpenAI-compatible provider" work per merge decision 6; + * it didn't → relay through the backend (``kind="code4me_backend"``), so + the backend chooses the upstream and also observes the call. + """ + from dataclasses import replace as _replace + + provider = self.adapter.provider + + if server.base_url: + provider = _replace( + provider, kind="openai", base_url=server.base_url + ) + elif backend_url: + provider = _replace( + provider, + kind="code4me_backend", + base_url=str(backend_url).rstrip("/"), + ) + + if server.model: + provider = _replace(provider, model=server.model) + if server.api_key_ref: + provider = _replace(provider, api_key_env=server.api_key_ref) + + memory_window = self.adapter.memory_window + if server.max_context_tokens is not None: + memory_window = _replace( + memory_window, max_tokens=server.max_context_tokens + ) + + adapter = _replace( + self.adapter, + provider=provider, + memory_window=memory_window, + max_iterations=server.max_iterations or self.adapter.max_iterations, + ) + + commands = self.commands + if server.commands_allowlist is not None: + commands = _replace( + commands, allowlisted_commands=list(server.commands_allowlist) + ) + + return _replace(self, adapter=adapter, commands=commands) + + @classmethod + def from_file(cls, config_path: str | Path) -> "AgentConfig": + path = Path(config_path).expanduser().resolve() + data = json.loads(path.read_text()) + + workspace_root_str = data.get("workspace_root", ".") + workspace_root = Path(str(workspace_root_str)).expanduser() + if not workspace_root.is_absolute(): + workspace_root = Path.cwd() / workspace_root + workspace_root = workspace_root.resolve() + + telemetry = data.get("telemetry", {}) + raw_trace_path = telemetry.get( + "trace_path", data.get("trace_path", ".code4me/acp-trace.jsonl") + ) + trace_path = Path(raw_trace_path).expanduser() + if not trace_path.is_absolute(): + trace_path = workspace_root / trace_path + + upload_data = telemetry.get("upload", {}) + raw_headers = upload_data.get("auth_headers", {}) + auth_headers: dict[str, str] = {} + if isinstance(raw_headers, dict): + auth_headers = { + str(header): str(value) + for header, value in raw_headers.items() + if header and value is not None + } + bearer_token = upload_data.get("bearer_token") + if bearer_token: + auth_headers["Authorization"] = f"Bearer {bearer_token}" + + raw_batch_size = upload_data.get("batch_size", 50) + try: + batch_size = max(1, int(raw_batch_size)) + except (TypeError, ValueError): + batch_size = 50 + + raw_timeout_seconds = upload_data.get("timeout_seconds", 5.0) + try: + timeout_seconds = float(raw_timeout_seconds) + if timeout_seconds <= 0: + timeout_seconds = 5.0 + except (TypeError, ValueError): + timeout_seconds = 5.0 + + raw_ingest_url = upload_data.get("ingest_url") + ingest_url = str(raw_ingest_url).strip() if raw_ingest_url else None + if not ingest_url: + ingest_url = None + + upload = UploadConfig( + enabled=bool(upload_data.get("enabled", False)), + ingest_url=ingest_url, + batch_size=batch_size, + timeout_seconds=timeout_seconds, + auth_headers=auth_headers, + ) + + commands_data = data.get("commands", data.get("command_tools", {})) + raw_allowlist = commands_data.get( + "allowlist", commands_data.get("allowlisted_commands", []) + ) + allowlisted_commands: list[str] = [] + if isinstance(raw_allowlist, list): + allowlisted_commands = [ + str(command).strip() + for command in raw_allowlist + if isinstance(command, str) and command.strip() + ] + + raw_command_timeout = commands_data.get("timeout_seconds", 10.0) + try: + command_timeout_seconds = float(raw_command_timeout) + if command_timeout_seconds <= 0: + command_timeout_seconds = 10.0 + except (TypeError, ValueError): + command_timeout_seconds = 10.0 + + raw_max_output_bytes = commands_data.get("max_output_bytes", 16384) + try: + max_output_bytes = max(1, int(raw_max_output_bytes)) + except (TypeError, ValueError): + max_output_bytes = 16384 + + commands = CommandConfig( + allowlisted_commands=allowlisted_commands, + timeout_seconds=command_timeout_seconds, + max_output_bytes=max_output_bytes, + ) + + adapter_data = data.get("adapter", {}) + if not isinstance(adapter_data, dict): + adapter_data = {} + + raw_max_iterations = adapter_data.get("max_iterations", 8) + try: + max_iterations = max(1, int(raw_max_iterations)) + except (TypeError, ValueError): + max_iterations = 8 + + memory_window_data = adapter_data.get("memory_window", {}) + if not isinstance(memory_window_data, dict): + memory_window_data = {} + memory_scope = str(memory_window_data.get("scope", "prompt")).strip() + if memory_scope not in {"prompt", "session"}: + memory_scope = "prompt" + memory_strategy = str( + memory_window_data.get("strategy", "last_messages") + ).strip() + if memory_strategy not in {"last_messages", "token_window"}: + memory_strategy = "last_messages" + raw_max_messages = memory_window_data.get("max_messages", 12) + try: + max_messages = max(1, int(raw_max_messages)) + except (TypeError, ValueError): + max_messages = 12 + raw_max_tokens = memory_window_data.get("max_tokens", 4000) + try: + max_tokens = max(1, int(raw_max_tokens)) + except (TypeError, ValueError): + max_tokens = 4000 + + fake_provider_data = adapter_data.get("fake_provider", {}) + if not isinstance(fake_provider_data, dict): + fake_provider_data = {} + raw_script = fake_provider_data.get("script", []) + script = raw_script if isinstance(raw_script, list) else [] + + provider_data = adapter_data.get("provider", {}) + if not isinstance(provider_data, dict): + provider_data = {} + raw_provider_headers = provider_data.get("auth_headers", {}) + provider_auth_headers: dict[str, str] = {} + if isinstance(raw_provider_headers, dict): + provider_auth_headers = { + str(header): str(value) + for header, value in raw_provider_headers.items() + if header and value is not None + } + + adapter = AdapterConfig( + name=str(adapter_data.get("name", "deterministic_echo")).strip() + or "deterministic_echo", + max_iterations=max_iterations, + memory_window=MemoryWindowConfig( + scope=memory_scope, + strategy=memory_strategy, + max_messages=max_messages, + max_tokens=max_tokens, + ), + fake_provider=FakeProviderConfig( + enabled=bool(fake_provider_data.get("enabled", False)), + script=script, + ), + # No provider defaults are invented here. Anything the config file + # omits stays empty and is filled from the backend's agent-config + # after authentication (see AgentConfig.with_server_overrides), so + # the assigned profile — not a local fallback — decides the model + # and endpoint. + provider=OpenAICompatibleProviderConfig( + kind=str(provider_data.get("kind", "code4me_backend")).strip() + or "code4me_backend", + base_url=str(provider_data.get("base_url", "")).strip(), + model=str(provider_data.get("model", "")).strip(), + api_key_env=str(provider_data.get("api_key_env", "")).strip(), + timeout_seconds=_positive_float( + provider_data.get("timeout_seconds", 90.0), + default=90.0, + ), + auth_headers=provider_auth_headers, + ), + ) + + return cls( + workspace_root=workspace_root, + trace_path=trace_path.resolve(), + session_id=data.get("session_id", uuid4().hex), + raw_capture_enabled=bool(telemetry.get("raw_capture_enabled", False)), + upload=upload, + commands=commands, + adapter=adapter, + ) + + +def _positive_float(value: object, *, default: float) -> float: + try: + parsed = float(value) + if parsed > 0: + return parsed + except (TypeError, ValueError): + pass + return default diff --git a/src/code4me2_agent/echo.py b/src/code4me2_agent/echo.py new file mode 100644 index 0000000..d44981e --- /dev/null +++ b/src/code4me2_agent/echo.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from time import perf_counter +from typing import TYPE_CHECKING +from uuid import uuid4 + +from code4me2_agent.adapters import MemoryWindow, create_agent_adapter +from code4me2_agent.command_tools import WorkspaceCommandTools +from code4me2_agent.file_tools import WorkspaceFileTools +from code4me2_agent.telemetry import AgentTelemetryRecorder + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from code4me2_agent.config import AgentConfig + from code4me2_agent.events import AgentEventSink + + +@dataclass(frozen=True) +class EchoPromptResult: + run_id: str + final_response: str + stop_reason: str + run_status: str + thoughts: tuple[str, ...] = () + duration_ms: float = 0.0 + + +class EchoAgentCore: + def __init__( + self, config: AgentConfig, *, event_sink: AgentEventSink | None = None + ) -> None: + self._config = config + self._event_sink = event_sink + self._telemetry = AgentTelemetryRecorder(config) + self.file_tools = WorkspaceFileTools(config, telemetry=self._telemetry) + self.command_tools = WorkspaceCommandTools(config, telemetry=self._telemetry) + self._session_memory = self._build_session_memory() + self._adapter = create_agent_adapter( + config, + telemetry=self._telemetry, + file_tools=self.file_tools, + command_tools=self.command_tools, + event_sink=self._event_sink, + ) + + def rebuild_tools( + self, + *, + acp_file_backend: object | None = None, + acp_command_backend: object | None = None, + ) -> None: + self.file_tools = WorkspaceFileTools( + self._config, + acp_backend=acp_file_backend, + telemetry=self._telemetry, + ) + self.command_tools = WorkspaceCommandTools( + self._config, + acp_backend=acp_command_backend, + telemetry=self._telemetry, + ) + self.rebuild_adapter() + + def rebuild_adapter(self) -> None: + self._adapter = create_agent_adapter( + self._config, + telemetry=self._telemetry, + file_tools=self.file_tools, + command_tools=self.command_tools, + event_sink=self._event_sink, + ) + + def _build_session_memory(self) -> MemoryWindow | None: + memory_config = self._config.adapter.memory_window + if memory_config.scope != "session": + return None + return MemoryWindow( + strategy=memory_config.strategy, + max_messages=memory_config.max_messages, + max_tokens=memory_config.max_tokens, + ) + + def load_session_memory(self, messages: list[dict]) -> None: + if self._session_memory is None: + return + self._session_memory.replace_messages(messages) + + def session_memory_snapshot(self) -> list[dict] | None: + if self._session_memory is None: + return None + return self._session_memory.snapshot() + + def handle_prompt( + self, + prompt: str, + request_id: str | None = None, + message_id: str | None = None, + run_id: str | None = None, + ) -> EchoPromptResult: + started_at = perf_counter() + request_id = request_id or uuid4().hex + run_id = run_id or uuid4().hex + run_start = self._telemetry.record( + event_type="agent.run.started", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={"status": "started"}, + ) + prompt_event = self._telemetry.record( + event_type="agent.request.received", + run_id=run_id, + request_id=request_id, + parent_event_id=run_start["event_id"], + message_id=message_id, + payload={"role": "user", "content": prompt}, + contains_user_prompt=True, + raw_payload={"role": "user", "content": prompt, "message_id": message_id}, + ) + + # 3 + logger.info("Adapter is: %s", self._adapter) + adapter_result = self._adapter.handle_prompt( + prompt=prompt, + run_id=run_id, + request_id=request_id, + message_id=message_id, + memory=self._session_memory, + ) + # n-1 + final_response = adapter_result.final_response + + response_event = self._telemetry.record( + event_type="agent.response.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=prompt_event["event_id"], + message_id=message_id, + payload={ + "role": "agent", + "content": final_response, + "stop_reason": adapter_result.stop_reason, + }, + metrics={"response_character_count": len(final_response)}, + contains_agent_response=True, + raw_payload={ + "role": "agent", + "content": final_response, + "message_id": message_id, + "stop_reason": adapter_result.stop_reason, + }, + ) + duration_ms = round((perf_counter() - started_at) * 1000, 3) + self._telemetry.record( + event_type="agent.run.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=response_event["event_id"], + payload={"status": adapter_result.run_status}, + metrics={ + "duration_ms": duration_ms, + "prompt_character_count": len(prompt), + "response_character_count": len(final_response), + }, + ) + return EchoPromptResult( + run_id=run_id, + final_response=final_response, + stop_reason=adapter_result.stop_reason, + run_status=adapter_result.run_status, + thoughts=adapter_result.thoughts, + duration_ms=duration_ms, + ) diff --git a/src/code4me2_agent/events.py b/src/code4me2_agent/events.py new file mode 100644 index 0000000..f4c7c64 --- /dev/null +++ b/src/code4me2_agent/events.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class ToolCallEvent: + phase: str + tool_call_id: str + tool_name: str + run_id: str + request_id: str + title: str + kind: str + status: str + path: str | None = None + content_text: str | None = None + raw_input: dict[str, Any] | None = None + raw_output: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class ThoughtEvent: + run_id: str + request_id: str + text: str + phase: str | None = None + duration_ms: float | None = None + + +class AgentEventSink(Protocol): + def tool_call(self, event: ToolCallEvent) -> None: + ... + + def thought(self, event: ThoughtEvent) -> None: + ... + + +class NoopAgentEventSink: + def tool_call(self, event: ToolCallEvent) -> None: + return + + def thought(self, event: ThoughtEvent) -> None: + return diff --git a/src/code4me2_agent/file_tools.py b/src/code4me2_agent/file_tools.py new file mode 100644 index 0000000..1bd6cbf --- /dev/null +++ b/src/code4me2_agent/file_tools.py @@ -0,0 +1,612 @@ +from __future__ import annotations + +import asyncio +import inspect +from dataclasses import dataclass +from difflib import unified_diff +from pathlib import Path +from time import perf_counter +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import run_awaitable_blocking +from code4me2_agent.telemetry import AgentTelemetryRecorder + +if TYPE_CHECKING: + from code4me2_agent.config import AgentConfig + + +@dataclass(frozen=True) +class FileReadResult: + path: str + content: str + backend_type: str + + +@dataclass(frozen=True) +class FileWriteResult: + path: str + bytes_written: int + backend_type: str + + +@dataclass(frozen=True) +class FileListResult: + paths: list[str] + backend_type: str + + +@dataclass(frozen=True) +class FileSearchResult: + matches: list[dict[str, object]] + backend_type: str + + +class AcpFileSystemBackend: + def __init__( + self, + *, + client: object, + session_id: str, + read_text_file_enabled: bool, + write_text_file_enabled: bool, + async_runner: object | None = None, + ) -> None: + self._client = client + self._session_id = session_id + self._read_text_file_enabled = read_text_file_enabled + self._write_text_file_enabled = write_text_file_enabled + self._async_runner = async_runner + + @property + def read_text_file_enabled(self) -> bool: + return self._read_text_file_enabled + + @property + def write_text_file_enabled(self) -> bool: + return self._write_text_file_enabled + + @property + def session_id(self) -> str: + return self._session_id + + def read_text_file(self, absolute_path: str) -> str: + response = self._run_client_call( + self._client.read_text_file( + path=absolute_path, + session_id=self._session_id, + limit=None, + line=None, + ) + ) + if isinstance(response, dict): + return str(response.get("content", "")) + return str(getattr(response, "content", "")) + + def write_text_file(self, absolute_path: str, content: str) -> None: + self._run_client_call( + self._client.write_text_file( + content=content, + path=absolute_path, + session_id=self._session_id, + ) + ) + + def _run_client_call(self, result: object) -> object: + if not inspect.isawaitable(result): + return result + if self._async_runner is not None: + return self._async_runner.run(result) + return run_awaitable_blocking(result) + + +def build_acp_file_system_backend( + *, + client: object, + session_id: str, + client_capabilities: object | None, + async_runner: object | None = None, +) -> AcpFileSystemBackend | None: + fs_capabilities = capability_value(client_capabilities, "fs") + read_enabled = bool( + capability_value(fs_capabilities, "readTextFile") + or capability_value(fs_capabilities, "read_text_file") + ) + write_enabled = bool( + capability_value(fs_capabilities, "writeTextFile") + or capability_value(fs_capabilities, "write_text_file") + ) + if not read_enabled and not write_enabled: + return None + return AcpFileSystemBackend( + client=client, + session_id=session_id, + read_text_file_enabled=read_enabled, + write_text_file_enabled=write_enabled, + async_runner=async_runner, + ) + + +class WorkspaceFileTools: + def __init__( + self, + config: AgentConfig, + acp_backend: object | None = None, + telemetry: AgentTelemetryRecorder | None = None, + ) -> None: + self._config = config + self._acp_backend = acp_backend + self._telemetry = telemetry or AgentTelemetryRecorder(config) + # TODO can we make these generic? + def read_file( + self, + path: str, + *, + line_start: int | None = None, + line_end: int | None = None, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileReadResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "read_file", + path, + tool_call_id, + run_id, + request_id, + started_at, + ) + relative_path = self._relative_path(resolved_path) + backend_type = "local" + read_text_file = self._acp_read_text_file() + if read_text_file is not None: + try: + content = read_text_file(str(resolved_path)) + backend_type = "acp" + except Exception as exc: # noqa: BLE001 + if not _is_missing_acp_session_error(exc): + raise + self._record_acp_backend_fallback( + tool_name="read_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + started_at=started_at, + error_message=str(exc), + operation="read_text_file", + ) + content = resolved_path.read_text(encoding="utf-8") + else: + content = resolved_path.read_text(encoding="utf-8") + content = _slice_lines(str(content), line_start=line_start, line_end=line_end) + self._record_tool_event( + tool_name="read_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + ) + return FileReadResult(path=relative_path, content=content, backend_type=backend_type) + + def create_file( + self, + path: str, + content: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileWriteResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "create_file", + path, + tool_call_id, + run_id, + request_id, + started_at, + ) + relative_path = self._relative_path(resolved_path) + backend_type = "local" + write_text_file = self._acp_write_text_file() + if write_text_file is None and resolved_path.exists(): + raise FileExistsError(f"File already exists: {path}") + if write_text_file is not None: + try: + write_text_file(str(resolved_path), content) + backend_type = "acp" + except Exception as exc: # noqa: BLE001 + if not _is_missing_acp_session_error(exc): + raise + self._record_acp_backend_fallback( + tool_name="create_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + started_at=started_at, + error_message=str(exc), + operation="write_text_file", + ) + if resolved_path.exists(): + raise FileExistsError(f"File already exists: {path}") from exc + resolved_path.parent.mkdir(parents=True, exist_ok=True) + resolved_path.write_text(content, encoding="utf-8") + else: + resolved_path.parent.mkdir(parents=True, exist_ok=True) + resolved_path.write_text(content, encoding="utf-8") + self._record_tool_event( + tool_name="create_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={ + "bytes_written": len(content.encode("utf-8")), + "content_capture_mode": self._content_capture_mode, + }, + raw_payload=self._raw_write_payload(relative_path, "", content), + ) + return FileWriteResult( + path=relative_path, + bytes_written=len(content.encode("utf-8")), + backend_type=backend_type, + ) + + def write_file( + self, + path: str, + content: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileWriteResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "write_file", + path, + tool_call_id, + run_id, + request_id, + started_at, + ) + before = resolved_path.read_text(encoding="utf-8") if resolved_path.exists() else "" + relative_path = self._relative_path(resolved_path) + backend_type = "local" + write_text_file = self._acp_write_text_file() + if write_text_file is not None: + try: + write_text_file(str(resolved_path), content) + backend_type = "acp" + except Exception as exc: # noqa: BLE001 + if not _is_missing_acp_session_error(exc): + raise + self._record_acp_backend_fallback( + tool_name="write_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + started_at=started_at, + error_message=str(exc), + operation="write_text_file", + ) + resolved_path.parent.mkdir(parents=True, exist_ok=True) + resolved_path.write_text(content, encoding="utf-8") + else: + resolved_path.parent.mkdir(parents=True, exist_ok=True) + resolved_path.write_text(content, encoding="utf-8") + bytes_written = len(content.encode("utf-8")) + self._record_tool_event( + tool_name="write_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={ + "bytes_written": bytes_written, + "content_capture_mode": self._content_capture_mode, + }, + raw_payload=self._raw_write_payload(relative_path, before, content), + ) + return FileWriteResult(path=relative_path, bytes_written=bytes_written, backend_type=backend_type) + + def replace_text( + self, + path: str, + old_text: str, + new_text: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileWriteResult: + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + current = self.read_file( + path, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + ).content + if old_text not in current: + raise ValueError("old_text was not found in the file.") + updated = current.replace(old_text, new_text, 1) + return self.write_file( + path, + updated, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def list_files( + self, + path: str = ".", + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileListResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "list_files", + path, + tool_call_id, + run_id, + request_id, + started_at, + ) + relative_path = self._relative_path(resolved_path) if resolved_path != self._config.workspace_root else "." + backend_type = "local" + files = [self._relative_path(candidate) for candidate in resolved_path.rglob("*") if candidate.is_file()] + files.sort() + self._record_tool_event( + tool_name="list_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={"path_count": len(files)}, + ) + return FileListResult(paths=files, backend_type=backend_type) + + def search_files( + self, + query: str, + *, + path: str = ".", + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileSearchResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "search_files", + path, + tool_call_id, + run_id, + request_id, + started_at, + ) + relative_path = self._relative_path(resolved_path) if resolved_path != self._config.workspace_root else "." + backend_type = "local" + matches: list[dict[str, object]] = [] + candidates = [resolved_path] if resolved_path.is_file() else sorted(resolved_path.rglob("*")) + for candidate in candidates: + if not candidate.is_file(): + continue + try: + lines = candidate.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError: + continue + for line_number, line in enumerate(lines, start=1): + if query in line: + matches.append( + { + "path": self._relative_path(candidate), + "line_number": line_number, + "line": line, + } + ) + self._record_tool_event( + tool_name="search_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={"query": query, "match_count": len(matches)}, + ) + return FileSearchResult(matches=matches, backend_type=backend_type) + + def _acp_read_text_file(self) -> Any | None: + if not bool(getattr(self._acp_backend, "read_text_file_enabled", False)): + return None + operation = getattr(self._acp_backend, "read_text_file", None) + if callable(operation): + return operation + return None + + def _acp_write_text_file(self) -> Any | None: + if not bool(getattr(self._acp_backend, "write_text_file_enabled", False)): + return None + operation = getattr(self._acp_backend, "write_text_file", None) + if callable(operation): + return operation + return None + + def _resolve_workspace_path(self, path: str) -> Path: + workspace_root = self._config.workspace_root.resolve() + requested_path = Path(path).expanduser() + if not requested_path.is_absolute(): + requested_path = workspace_root / requested_path + resolved_path = requested_path.resolve() + if resolved_path != workspace_root and workspace_root not in resolved_path.parents: + raise PermissionError("Path is outside the configured workspace root.") + return resolved_path + + def _resolve_or_record_denial( + self, + tool_name: str, + path: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> Path: + try: + return self._resolve_workspace_path(path) + except PermissionError: + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=path, + status="denied", + backend_type="local", + started_at=started_at, + denial_reason="outside_workspace_root", + ) + raise + + def _relative_path(self, path: Path) -> str: + return path.relative_to(self._config.workspace_root).as_posix() + + @property + def _content_capture_mode(self) -> str: + return "raw" if self._config.raw_capture_enabled else "redacted" + + def _raw_write_payload(self, relative_path: str, before: str, after: str) -> dict[str, str]: + return { + "content": after, + "diff": "".join( + unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=relative_path, + tofile=relative_path, + ) + ), + } + + def _record_tool_event( + self, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + path: str, + status: str, + backend_type: str, + started_at: float, + denial_reason: str | None = None, + extra_payload: dict[str, object] | None = None, + raw_payload: dict[str, object] | None = None, + ) -> None: + event_type = "agent.tool.denied" if status == "denied" else "agent.tool.completed" + payload = { + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "backend_type": backend_type, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "status": status, + "path": path, + } + payload.update(extra_payload or {}) + if denial_reason is not None: + payload["denial_reason"] = denial_reason + self._telemetry.record( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload=payload, + raw_payload=raw_payload, + ) + + def _record_acp_backend_fallback( + self, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + path: str, + started_at: float, + error_message: str, + operation: str, + ) -> None: + self._telemetry.record( + event_type="agent.acp.backend_fallback", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "path": path, + "backend_type": "acp", + "fallback_backend_type": "local", + "acp_session_id": getattr(self._acp_backend, "session_id", None), + "operation": operation, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "error_message": error_message, + }, + ) + + +def _is_missing_acp_session_error(exc: Exception) -> bool: + message = str(exc).lower() + return "session" in message and "not found" in message + + +def _slice_lines( + content: str, + *, + line_start: int | None = None, + line_end: int | None = None, +) -> str: + if line_start is None and line_end is None: + return content + + lines = content.splitlines(keepends=True) + start_index = 0 if line_start is None else max(0, int(line_start) - 1) + end_index = len(lines) if line_end is None else max(start_index, int(line_end)) + return "".join(lines[start_index:end_index]) diff --git a/src/code4me2_agent/runtime_auth.py b/src/code4me2_agent/runtime_auth.py new file mode 100644 index 0000000..f1a4824 --- /dev/null +++ b/src/code4me2_agent/runtime_auth.py @@ -0,0 +1,588 @@ +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib import error, request + +logger = logging.getLogger(__name__) + + +def _log_secrets_enabled() -> bool: + return os.environ.get("CODE4ME_ACP_LOG_SECRETS", "").lower() in {"1", "true", "yes"} + + +def _secret_for_log(value: str | None) -> str: + if not value: + return "missing" + if _log_secrets_enabled(): + return value + return "present" + + +class AcpAuthorizationFailure(Exception): + pass + + +@dataclass(frozen=True) +class AcpRuntimeScope: + project_id: str + workspace: str + + +class AcpBackendAuthorization: + def __init__( + self, + *, + backend_url: str | None, + grant: str | None, + acp_token: str | None = None, + workspace_root: str | Path | None = None, + ) -> None: + self._backend_url = backend_url.rstrip("/") if backend_url else None + self._grant = grant + self._acp_token: str | None = acp_token + self._scope: AcpRuntimeScope | None = None + self._workspace_root = ( + Path(workspace_root).resolve() if workspace_root is not None else None + ) + + @classmethod + def from_environment( + cls, + *, + workspace_root: str | Path | None = None, + ) -> "AcpBackendAuthorization": + backend_url = os.environ.get("CODE4ME_ACP_BACKEND_URL") + grant = os.environ.get("CODE4ME_ACP_GRANT") + acp_token = os.environ.get("CODE4ME_ACP_TOKEN") + if ( + not backend_url or not grant or not acp_token + ) and workspace_root is not None: + logger.info( + "Loading prepared ACP runtime credentials from workspace handoff file." + ) + fallback_values = _read_runtime_auth_values( + Path(workspace_root), include_home=True + ) + backend_url = backend_url or fallback_values.get("CODE4ME_ACP_BACKEND_URL") + grant = grant or fallback_values.get("CODE4ME_ACP_GRANT") + acp_token = acp_token or fallback_values.get("CODE4ME_ACP_TOKEN") + logger.info( + "Prepared ACP runtime credentials loaded: backend_url=%s grant=%s acp_token=%s workspace_root=%s.", + backend_url or "missing", + _secret_for_log(grant), + _secret_for_log(acp_token), + Path(workspace_root).resolve() if workspace_root is not None else "missing", + ) + return cls( + backend_url=backend_url, + grant=grant, + acp_token=acp_token, + workspace_root=workspace_root, + ) + + @property + def is_authenticated(self) -> bool: + return self._acp_token is not None and self._scope is not None + + @property + def backend_url(self) -> str | None: + return self._backend_url + + def authenticate(self) -> AcpRuntimeScope: + self._refresh_prepared_credentials() + if self._backend_url and self._acp_token: + try: + return self.validate() + except AcpAuthorizationFailure: + logger.info( + "Cached ACP runtime token was not valid; falling back to prepared grant exchange." + ) + if not self._backend_url or not self._grant: + logger.warning( + "ACP authentication failed because prepared credentials are missing: backend_url=%s grant=%s workspace_root=%s.", + self._backend_url or "missing", + _secret_for_log(self._grant), + self._workspace_root or "missing", + ) + raise AcpAuthorizationFailure( + "Prepared Code4Me runtime credentials are required." + ) + logger.info( + "Exchanging prepared ACP runtime grant with backend_url=%s grant=%s.", + self._backend_url, + _secret_for_log(self._grant), + ) + payload = self._post("/api/acp/session/exchange", {"grant": self._grant}) + self._grant = None + acp_token = payload.get("acp_token") + project_id = payload.get("project_id") + workspace = payload.get("workspace") + if not acp_token or not project_id or not workspace: + logger.warning( + "ACP grant exchange returned an invalid authorization scope." + ) + raise AcpAuthorizationFailure( + "ACP grant exchange returned an invalid authorization scope." + ) + self._acp_token = str(acp_token) + self._scope = AcpRuntimeScope( + project_id=str(project_id), workspace=str(workspace) + ) + self._write_session_cache() + self.fetch_agent_config() + logger.info( + "ACP authentication succeeded for project_id=%s workspace=%s acp_token=%s.", + project_id, + workspace, + _secret_for_log(self._acp_token), + ) + return self._scope + + def validate(self) -> AcpRuntimeScope: + if not self._backend_url or not self._acp_token: + logger.warning( + "ACP validation failed because authentication state is missing." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + try: + logger.info( + "Validating ACP runtime authentication with backend_url=%s acp_token=%s.", + self._backend_url, + _secret_for_log(self._acp_token), + ) + payload = self._post( + "/api/acp/session/validate-or-refresh", + {}, + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + except AcpAuthorizationFailure: + self._acp_token = None + self._scope = None + logger.warning( + "ACP validation failed; cleared runtime authentication state." + ) + raise + project_id = payload.get("project_id") + workspace = payload.get("workspace") + if not project_id or not workspace: + self._acp_token = None + self._scope = None + logger.warning("ACP validation returned an invalid authorization scope.") + raise AcpAuthorizationFailure( + "ACP validation returned an invalid authorization scope." + ) + self._scope = AcpRuntimeScope( + project_id=str(project_id), workspace=str(workspace) + ) + logger.info( + "ACP validation succeeded for project_id=%s workspace=%s.", + project_id, + workspace, + ) + return self._scope + + # The assigned agent profile, as handed down by the backend after auth. + # Held as one object rather than a field per setting so adding a profile + # setting doesn't require touching this class at all. + _server_agent_config: "ServerAgentConfig | None" = None + + @property + def server_agent_config(self) -> "ServerAgentConfig | None": + """The backend's assigned configuration, or None if it couldn't be fetched.""" + return self._server_agent_config + + # Backwards-compatible accessors, kept so existing callers and tests that + # read individual settings keep working. + @property + def server_commands_allowlist(self) -> list[str] | None: + return ( + self._server_agent_config.commands_allowlist + if self._server_agent_config + else None + ) + + @property + def server_model(self) -> str | None: + return self._server_agent_config.model if self._server_agent_config else None + + @property + def server_max_iterations(self) -> int | None: + return ( + self._server_agent_config.max_iterations + if self._server_agent_config + else None + ) + + @property + def server_base_url(self) -> str | None: + return ( + self._server_agent_config.base_url if self._server_agent_config else None + ) + + @property + def server_api_key_ref(self) -> str | None: + return ( + self._server_agent_config.api_key_ref + if self._server_agent_config + else None + ) + + @property + def server_store_agent_content(self) -> bool: + """Whether the server will persist content for this user. + + Advisory only. The server enforces its own decision regardless of what + this agent sends, so this is used purely to avoid transmitting content + that would be discarded — never to enable capture. + """ + return ( + self._server_agent_config.store_agent_content + if self._server_agent_config + else True + ) + + def fetch_agent_config(self) -> None: + """Fetch the assigned agent profile's runtime configuration. + + Called right after a successful grant exchange. A failure here is + logged and tolerated: the agent then runs on whatever the local config + file provides, which is a degraded but working state rather than a dead + one. Note that with the provider defaults removed, a local config that + specifies no model will fail loudly at first use instead — which is the + intended outcome, since silently substituting a different model would + corrupt the experiment. + """ + if not self._backend_url or not self._acp_token: + logger.info("Cannot fetch agent config: not authenticated.") + return + try: + payload = self._get( + "/api/acp/agent-config", + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + except AcpAuthorizationFailure: + logger.info("Failed to fetch server agent config; using local config.") + return + + from code4me2_agent.config import ServerAgentConfig + + self._server_agent_config = ServerAgentConfig.from_payload(payload) + config = self._server_agent_config + logger.info( + "Server agent config: profile=%s runtime=%s model=%s base_url=%s " + "api_key_ref=%s tools=%s max_iterations=%s store_content=%s.", + config.agent_profile, + config.framework_version, + config.model, + config.base_url or "(via backend relay)", + config.api_key_ref, + config.tools, + config.max_iterations, + config.store_agent_content, + ) + + def telemetry_headers(self) -> dict[str, str]: + if not self._acp_token: + logger.warning( + "Telemetry headers requested before ACP authentication completed." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + return {"Authorization": f"Bearer {self._acp_token}"} + + def authorized_json_request( + self, + method: str, + path: str, + payload: dict | None = None, + *, + timeout: float = 10.0, + ) -> dict: + if not self._backend_url or not self._acp_token: + logger.warning( + "ACP authorized request failed because authentication state is missing." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + data = None if payload is None else json.dumps(payload).encode("utf-8") + http_request = request.Request( + f"{self._backend_url}{path}", + data=data, + headers={ + "Authorization": f"Bearer {self._acp_token}", + "Content-Type": "application/json", + }, + method=method, + ) + logger.info( + "Sending ACP authorized request method=%s url=%s payload_keys=%s authorization_header=%s.", + method, + http_request.full_url, + sorted(payload.keys()) if payload else [], + _secret_for_log(f"Bearer {self._acp_token}"), + ) + try: + with request.urlopen(http_request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + if exc.code == 404: + return {} + logger.warning( + "ACP authorized request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected." + ) from None + except error.URLError as exc: + logger.warning( + "ACP authorized request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected." + ) from None + except TimeoutError: + logger.warning("ACP authorized request timed out.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected." + ) from None + except ValueError: + logger.warning("ACP authorized request returned invalid JSON.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected." + ) from None + + def _refresh_prepared_credentials(self) -> None: + if self._workspace_root is None: + return + fallback_values = _read_runtime_auth_values( + self._workspace_root, include_home=True + ) + fallback_backend_url = fallback_values.get("CODE4ME_ACP_BACKEND_URL") + fallback_grant = fallback_values.get("CODE4ME_ACP_GRANT") + fallback_acp_token = fallback_values.get("CODE4ME_ACP_TOKEN") + if fallback_backend_url: + self._backend_url = fallback_backend_url.rstrip("/") + logger.info("Refreshed ACP backend URL from handoff file.") + if fallback_grant: + self._grant = fallback_grant + logger.info("Refreshed ACP runtime grant from handoff file.") + if fallback_acp_token: + self._acp_token = fallback_acp_token + logger.info("Refreshed ACP runtime token from handoff file.") + + def _write_session_cache(self) -> None: + if not self._backend_url or not self._acp_token: + return + cache_path = Path("~/.code4me/acp-session.env").expanduser() + workspace_cache_path = ( + self._workspace_root / ".code4me" / "acp-runtime.env" + if self._workspace_root is not None + else None + ) + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text( + "CODE4ME_ACP_BACKEND_URL=" + f"{self._backend_url}\n" + "CODE4ME_ACP_TOKEN=" + f"{self._acp_token}\n", + encoding="utf-8", + ) + os.chmod(cache_path, 0o600) + logger.info("Cached ACP runtime session token at %s.", cache_path) + except OSError as exc: + logger.info( + "Could not cache ACP runtime session token at %s: %s.", cache_path, exc + ) + if workspace_cache_path is not None: + try: + _write_runtime_auth_values( + workspace_cache_path, + { + "CODE4ME_ACP_BACKEND_URL": self._backend_url, + "CODE4ME_ACP_TOKEN": self._acp_token, + }, + ) + os.chmod(workspace_cache_path, 0o600) + logger.info( + "Cached ACP runtime session token in workspace handoff at %s.", + workspace_cache_path, + ) + except OSError as exc: + logger.info( + "Could not cache ACP runtime session token in workspace handoff at %s: %s.", + workspace_cache_path, + exc, + ) + + def _post( + self, + path: str, + payload: dict[str, str], + *, + headers: dict[str, str] | None = None, + ) -> dict: + http_request = request.Request( + f"{self._backend_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", **(headers or {})}, + method="POST", + ) + logger.info( + "Sending ACP authorization request method=POST url=%s payload_keys=%s authorization_header=%s.", + http_request.full_url, + sorted(payload.keys()), + _secret_for_log(headers.get("Authorization") if headers else None), + ) + try: + with request.urlopen(http_request, timeout=10.0) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + logger.warning( + "ACP authorization request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except error.URLError as exc: + logger.warning( + "ACP authorization request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except TimeoutError: + logger.warning("ACP authorization request timed out.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except ValueError: + logger.warning("ACP authorization request returned invalid JSON.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + + def _get( + self, + path: str, + *, + headers: dict[str, str] | None = None, + ) -> dict: + http_request = request.Request( + f"{self._backend_url}{path}", + headers={"Content-Type": "application/json", **(headers or {})}, + method="GET", + ) + logger.info( + "Sending ACP agent-config request method=GET url=%s authorization_header=%s.", + http_request.full_url, + _secret_for_log(headers.get("Authorization") if headers else None), + ) + try: + with request.urlopen(http_request, timeout=10.0) as response: + parsed = json.loads(response.read().decode("utf-8")) + return parsed if isinstance(parsed, dict) else {} + except error.HTTPError as exc: + logger.warning( + "ACP agent-config request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except error.URLError as exc: + logger.warning( + "ACP agent-config request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except TimeoutError: + logger.warning("ACP agent-config request timed out.") + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except ValueError: + logger.warning("ACP agent-config request returned invalid JSON.") + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + + +def _read_env_file(file_path: Path) -> dict[str, str]: + try: + raw_lines = file_path.read_text().splitlines() + except OSError: + logger.info("ACP runtime env file was not available at %s.", file_path) + return {} + values: dict[str, str] = {} + for line in raw_lines: + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if key in {"CODE4ME_ACP_BACKEND_URL", "CODE4ME_ACP_GRANT", "CODE4ME_ACP_TOKEN"}: + values[key] = value.strip() + return values + + +def _write_runtime_auth_values(file_path: Path, values: dict[str, str]) -> None: + existing_values = _read_env_file(file_path) + merged_values = {**existing_values, **values} + file_path.parent.mkdir(parents=True, exist_ok=True) + payload = "".join( + f"{key}={value}\n" + for key, value in merged_values.items() + if key in {"CODE4ME_ACP_BACKEND_URL", "CODE4ME_ACP_GRANT", "CODE4ME_ACP_TOKEN"} + ) + file_path.write_text(payload, encoding="utf-8") + + +def _read_runtime_auth_values( + workspace_root: Path, *, include_home: bool = False +) -> dict[str, str]: + candidates = [ + workspace_root.resolve() / ".idea" / "code4me" / "acp-runtime.env", + workspace_root.resolve() / ".code4me" / "acp-runtime.env", + ] + if include_home: + candidates.extend( + [ + Path("~/.code4me/acp-runtime.env").expanduser(), + Path("~/.code4me/acp-session.env").expanduser(), + ] + ) + available: list[tuple[float, Path, dict[str, str]]] = [] + for handoff_path in candidates: + values = _read_env_file(handoff_path) + if values: + try: + modified_at = handoff_path.stat().st_mtime + except OSError: + modified_at = 0.0 + available.append((modified_at, handoff_path, values)) + continue + logger.info("ACP runtime handoff file was not available at %s.", handoff_path) + if available: + merged: dict[str, str] = {} + for _, handoff_path, values in sorted( + available, key=lambda item: item[0], reverse=True + ): + logger.info("Considering ACP runtime handoff file at %s.", handoff_path) + for key, value in values.items(): + merged.setdefault(key, value) + return merged + return {} + + +def _read_runtime_handoff_file( + workspace_root: Path, *, include_home: bool = False +) -> tuple[str | None, str | None]: + values = _read_runtime_auth_values(workspace_root, include_home=include_home) + return values.get("CODE4ME_ACP_BACKEND_URL"), values.get("CODE4ME_ACP_GRANT") diff --git a/src/code4me2_agent/telemetry.py b/src/code4me2_agent/telemetry.py new file mode 100644 index 0000000..6ca278b --- /dev/null +++ b/src/code4me2_agent/telemetry.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Protocol +from urllib import request +from uuid import uuid4 + +if TYPE_CHECKING: + from pathlib import Path + + from code4me2_agent.config import AgentConfig + + +SCHEMA_VERSION = "code4me.agent.event.v1" + +# TODO this file should be working with the database. and may be moved to another folder. +class TelemetrySink(Protocol): + def append(self, event: dict[str, Any]) -> None: + ... + + +class UploadEventsCallable(Protocol): + def __call__( + self, + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + ) -> None: + ... + + +class JsonlTelemetrySink: + def __init__(self, trace_path: Path) -> None: + self._trace_path = trace_path + + def append(self, event: dict[str, Any]) -> None: + self._trace_path.parent.mkdir(parents=True, exist_ok=True) + with self._trace_path.open("a", encoding="utf-8") as trace_file: + trace_file.write(json.dumps(event, sort_keys=True)) + trace_file.write("\n") + + +JsonlTraceRecorder = JsonlTelemetrySink + + +class ServerUploadTelemetrySink: + def __init__( + self, + upload_events: UploadEventsCallable, + batch_size: int = 50, + auth_headers: dict[str, str] | None = None, + ) -> None: + self._upload_events = upload_events + self._batch_size = max(1, int(batch_size)) + self._auth_headers = dict(auth_headers or {}) + self._pending_run_events: dict[str, list[dict[str, Any]]] = {} + + def append(self, event: dict[str, Any]) -> None: + run_id = event["run_id"] + pending_events = self._pending_run_events.setdefault(run_id, []) + pending_events.append(event) + self._flush_ready_runs() + + def _flush_ready_runs(self) -> None: + for run_id in list(self._pending_run_events.keys()): + pending_events = self._pending_run_events[run_id] + if not self._should_flush(pending_events): + continue + + run_payload = _build_run_payload(pending_events) + try: + self._upload_events(run_payload, pending_events, dict(self._auth_headers)) + except Exception: + continue + del self._pending_run_events[run_id] + + def _should_flush(self, pending_events: list[dict[str, Any]]) -> bool: + if len(pending_events) >= self._batch_size: + return True + return any(event["event_type"] == "agent.run.completed" for event in pending_events) + + +def _build_run_payload(events: list[dict[str, Any]]) -> dict[str, Any]: + first_event = events[0] + status = "in_progress" + started_at = None + completed_at = None + + for event in events: + if event["event_type"] == "agent.run.started": + status = str(event.get("payload", {}).get("status", "started")) + started_at = event["timestamp"] + elif event["event_type"] == "agent.run.completed": + status = str(event.get("payload", {}).get("status", "completed")) + completed_at = event["timestamp"] + + return { + "run_id": first_event["run_id"], + "session_id": first_event["session_id"], + "status": status, + "source": first_event.get("source", "code4me2_agent"), + "started_at": started_at, + "completed_at": completed_at, + } + + +def upload_event_batch_http( + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + *, + ingest_url: str, + timeout_seconds: float, +) -> None: + payload = {"run": run, "events": events} + payload_bytes = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + headers.update(auth_headers) + http_request = request.Request( + ingest_url, + data=payload_bytes, + headers=headers, + method="POST", + ) + with request.urlopen(http_request, timeout=timeout_seconds) as response: + if response.status >= 400: + raise RuntimeError(f"Telemetry upload failed with HTTP {response.status}.") + + +def build_upload_sink(config: AgentConfig) -> ServerUploadTelemetrySink | None: + if not config.upload.enabled or config.upload.ingest_url is None: + return None + + def upload_events( + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + ) -> None: + upload_event_batch_http( + run=run, + events=events, + auth_headers=auth_headers, + ingest_url=config.upload.ingest_url, + timeout_seconds=config.upload.timeout_seconds, + ) + + return ServerUploadTelemetrySink( + upload_events=upload_events, + batch_size=config.upload.batch_size, + auth_headers=config.upload.auth_headers, + ) + + +class AgentTelemetryRecorder: + def __init__(self, config: AgentConfig, sinks: list[TelemetrySink] | None = None) -> None: + self._config = config + self._jsonl = JsonlTelemetrySink(config.trace_path) + if sinks is None: + upload_sink = build_upload_sink(config) + self._sinks = [upload_sink] if upload_sink is not None else [] + else: + self._sinks = sinks + self._run_sequences: dict[str, int] = {} + + def record( + self, + *, + event_type: str, + run_id: str, + request_id: str, + parent_event_id: str | None, + payload: dict[str, Any], + message_id: str | None = None, + metrics: dict[str, Any] | None = None, + contains_user_prompt: bool = False, + contains_agent_response: bool = False, + raw_payload: dict[str, Any] | None = None, + ) -> dict[str, Any]: + sequence = self._run_sequences.get(run_id, 0) + 1 + self._run_sequences[run_id] = sequence + + observed_tool = str(payload.get("tool_name", "none")) + event = { + "schema_version": SCHEMA_VERSION, + "event_id": uuid4().hex, + "event_type": event_type, + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "sequence": sequence, + "source": "code4me2_agent", + "session_id": self._config.session_id, + "run_id": run_id, + "request_id": request_id, + "parent_event_id": parent_event_id, + "payload": payload, + "raw_payload": raw_payload if self._config.raw_capture_enabled else None, + "metrics": metrics or {}, + "privacy": { + "capture_mode": "raw" if self._config.raw_capture_enabled else "redacted", + "raw_capture_enabled": self._config.raw_capture_enabled, + "contains_user_prompt": contains_user_prompt, + "contains_agent_response": contains_agent_response, + "redaction_applied": not self._config.raw_capture_enabled, + }, + "observability": { + "level": "full", + "adapter": self._config.adapter.name, + "llm": self._config.adapter.name, + "tools": observed_tool, + "missing_internals": [], + }, + } + if message_id is not None: + event["message_id"] = message_id + + self._jsonl.append(event) + for sink in self._sinks: + sink.append(event) + return event diff --git a/src/database/crud.py b/src/database/crud.py index 6bc0e24..b0aad8c 100644 --- a/src/database/crud.py +++ b/src/database/crud.py @@ -1183,3 +1183,707 @@ def get_documentation_stats(db: Session) -> dict: ), "languages": dict(language_stats), } + + +# ── Agent CRUD ──────────────────────────────────────────────────────────────── +# +# Content parameters (task_description, first_system_message, last_user_message, +# response_text, tool_arguments, tool_result, payload_json, diff_text, +# edit_delta_json) are optional and default to None. Callers are responsible for +# passing None unless the user's store_agent_content preference resolves True — +# see backend.routers.agent.consent.resolve_store_agent_content. These helpers +# deliberately do not re-check consent, so there is exactly one place where that +# decision is made. + + +class DuplicateAgentEventError(ValueError): + """Raised when a self-reported event batch replays already-ingested ids. + + Ingestion is idempotent by ``span_id`` (the runtime's own event id), so a + retried upload is rejected rather than duplicated. + """ + + def __init__(self, event_ids: List[str]) -> None: + self.event_ids = event_ids + super().__init__(f"Duplicate agent event IDs: {', '.join(event_ids)}") + + +def create_agent_profile( + db: Session, + name: str, + model: str, + tools_json: str, + approval_policy: str, + max_steps: int, + framework_version: str = "code4me2-agent", + base_url: Optional[str] = None, + api_key_ref: Optional[str] = None, + is_active: bool = True, + max_context_tokens: Optional[int] = None, + temperature: Optional[float] = None, +) -> db_schemas.AgentProfile: + profile = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), + name=name, + model=model, + framework_version=framework_version, + base_url=base_url, + api_key_ref=api_key_ref, + tools_json=tools_json, + approval_policy=approval_policy, + max_steps=max_steps, + is_active=is_active, + max_context_tokens=max_context_tokens, + temperature=temperature, + ) + db.add(profile) + db.commit() + db.refresh(profile) + return profile + + +def get_agent_profile(db: Session, name: str) -> Optional[db_schemas.AgentProfile]: + return ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.name == name) + .first() + ) + + +def get_agent_profile_by_id( + db: Session, profile_id: uuid.UUID +) -> Optional[db_schemas.AgentProfile]: + return ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .first() + ) + + +def list_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: + return db.query(db_schemas.AgentProfile).order_by(db_schemas.AgentProfile.name).all() + + +def list_active_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: + """Candidate arms for new A/B assignments — only profiles flagged active. + + Ordered deterministically by name so behaviour is reproducible across calls + (relevant for tests and for any future deterministic bucketing). + """ + return ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.is_active.is_(True)) + .order_by(db_schemas.AgentProfile.name) + .all() + ) + + +def update_agent_profile( + db: Session, + profile_id: uuid.UUID, + name: str, + model: str, + tools_json: str, + approval_policy: str, + max_steps: int, + framework_version: str = "code4me2-agent", + base_url: Optional[str] = None, + api_key_ref: Optional[str] = None, + is_active: bool = True, + max_context_tokens: Optional[int] = None, + temperature: Optional[float] = None, +) -> Optional[db_schemas.AgentProfile]: + result = ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .update( + { + "name": name, + "model": model, + "framework_version": framework_version, + "base_url": base_url, + "api_key_ref": api_key_ref, + "tools_json": tools_json, + "approval_policy": approval_policy, + "max_steps": max_steps, + "is_active": is_active, + "max_context_tokens": max_context_tokens, + "temperature": temperature, + } + ) + ) + db.commit() + if result: + return get_agent_profile_by_id(db, profile_id) + return None + + +def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: + result = ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .delete() + ) + db.commit() + return result > 0 + + +# ── Agent profile assignments (A/B testing) ───────────────────────────────── + + +def get_agent_profile_assignment( + db: Session, user_id: uuid.UUID +) -> Optional[db_schemas.AgentProfileAssignment]: + return ( + db.query(db_schemas.AgentProfileAssignment) + .filter(db_schemas.AgentProfileAssignment.user_id == user_id) + .first() + ) + + +def set_agent_profile_assignment( + db: Session, + user_id: uuid.UUID, + profile_id: uuid.UUID, + source: str = "auto", +) -> db_schemas.AgentProfileAssignment: + """Upsert a user's profile assignment. + + Used both for the first-contact random draw (source="auto") and for admin + overrides (source="manual"). Re-assigning overwrites the existing row. + """ + assignment = get_agent_profile_assignment(db, user_id) + if assignment is None: + assignment = db_schemas.AgentProfileAssignment( + user_id=user_id, + profile_id=profile_id, + source=source, + ) + db.add(assignment) + else: + assignment.profile_id = profile_id + assignment.source = source + assignment.assigned_at = datetime.now() + db.commit() + db.refresh(assignment) + return assignment + + +def delete_agent_profile_assignment(db: Session, user_id: uuid.UUID) -> bool: + result = ( + db.query(db_schemas.AgentProfileAssignment) + .filter(db_schemas.AgentProfileAssignment.user_id == user_id) + .delete() + ) + db.commit() + return result > 0 + + +def list_agent_profile_assignments( + db: Session, +) -> List[db_schemas.AgentProfileAssignment]: + return db.query(db_schemas.AgentProfileAssignment).all() + + +# ── Study ↔ agent profile links (A/B studies for agents) ──────────────────── + + +def set_study_agent_profiles( + db: Session, + study_id: uuid.UUID, + profile_ids: List[uuid.UUID], + baseline_profile_id: Optional[uuid.UUID] = None, +) -> None: + """Replace the agent-profile arms attached to a study. + + Idempotent: clears any existing rows for the study and writes one row per + profile, flagging ``baseline_profile_id`` (if given) as the baseline arm. + Does not commit — the caller owns the transaction (study create/activate + writes several tables atomically). + """ + db.query(db_schemas.StudyAgentProfile).filter( + db_schemas.StudyAgentProfile.study_id == study_id + ).delete() + for profile_id in profile_ids: + db.add( + db_schemas.StudyAgentProfile( + study_id=study_id, + profile_id=profile_id, + is_baseline=(profile_id == baseline_profile_id), + ) + ) + + +def list_study_agent_profiles( + db: Session, study_id: uuid.UUID +) -> List[db_schemas.StudyAgentProfile]: + """All agent-profile arm links for a study (with the joined profile loaded).""" + return ( + db.query(db_schemas.StudyAgentProfile) + .filter(db_schemas.StudyAgentProfile.study_id == study_id) + .all() + ) + + +def list_active_study_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: + """Candidate arms drawn from the currently-active study's selected profiles. + + Returns only profiles that are both attached to the active study and still + flagged ``is_active``. Empty when there is no active study, or the active + study selected no agent profiles — callers fall back to + ``list_active_agent_profiles`` in that case (backwards compatible). + Ordered by name for reproducibility, matching ``list_active_agent_profiles``. + """ + return ( + db.query(db_schemas.AgentProfile) + .join( + db_schemas.StudyAgentProfile, + db_schemas.StudyAgentProfile.profile_id + == db_schemas.AgentProfile.profile_id, + ) + .join( + db_schemas.Study, + db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, + ) + .filter(db_schemas.Study.is_active.is_(True)) + .filter(db_schemas.AgentProfile.is_active.is_(True)) + .order_by(db_schemas.AgentProfile.name) + .all() + ) + + +# ── Agent tasks ───────────────────────────────────────────────────────────── + + +def create_agent_task( + db: Session, + agent_profile: str, + model: str, + approval_policy: str, + tools_json: str, + task_description: Optional[str] = None, + session_id: Optional[uuid.UUID] = None, + task_id: Optional[uuid.UUID] = None, + temperature: Optional[float] = None, + framework_version: Optional[str] = None, + source: str = "plugin", + owner_user_id: Optional[uuid.UUID] = None, + owner_project_id: Optional[uuid.UUID] = None, + external_run_id: Optional[str] = None, + agent_session_id: Optional[str] = None, + status: str = "pending", + started_at: Optional[datetime] = None, +) -> db_schemas.AgentTask: + task = db_schemas.AgentTask( + task_id=task_id or uuid.uuid4(), + agent_profile=agent_profile, + model=model, + temperature=temperature, + approval_policy=approval_policy, + tools_json=tools_json, + framework_version=framework_version, + status=status, + source=source, + session_id=session_id, + owner_user_id=owner_user_id, + owner_project_id=owner_project_id, + external_run_id=external_run_id, + agent_session_id=agent_session_id, + started_at=started_at, + task_description=task_description, + ) + db.add(task) + db.commit() + db.refresh(task) + return task + + +def get_agent_task(db: Session, task_id: uuid.UUID) -> Optional[db_schemas.AgentTask]: + return ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.task_id == task_id) + .first() + ) + + +def get_agent_task_by_external_run_id( + db: Session, external_run_id: str +) -> Optional[db_schemas.AgentTask]: + """Look a task up by the *runtime's* own run id. + + The self-reporting ``code4me2-agent`` runtime mints its own run id before it + ever talks to the backend, so ingestion resolves the task this way rather + than requiring the runtime to adopt a server-issued UUID. + """ + return ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.external_run_id == external_run_id) + .first() + ) + + +def get_open_agent_tasks_for_session( + db: Session, session_id: uuid.UUID +) -> List[db_schemas.AgentTask]: + """Tasks belonging to this session that haven't reached a terminal state yet.""" + return ( + db.query(db_schemas.AgentTask) + .filter( + db_schemas.AgentTask.session_id == session_id, + db_schemas.AgentTask.status.notin_(["done", "failed"]), + ) + .all() + ) + + +def update_agent_task_status( + db: Session, + task_id: uuid.UUID, + status: str, + completed_at: Optional[datetime] = None, + total_steps: Optional[int] = None, + input_tokens: Optional[int] = None, + output_tokens: Optional[int] = None, + latest_event_id: Optional[uuid.UUID] = None, +) -> bool: + data: dict = {"status": status} + if completed_at is not None: + data["completed_at"] = completed_at + if total_steps is not None: + data["total_steps"] = total_steps + if input_tokens is not None: + data["input_tokens"] = input_tokens + if output_tokens is not None: + data["output_tokens"] = output_tokens + if latest_event_id is not None: + data["latest_event_id"] = latest_event_id + result = ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.task_id == task_id) + .update(data) + ) + db.commit() + return result > 0 + + +def update_agent_task_tools( + db: Session, + task_id: uuid.UUID, + tools: List[str], +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"tools_json": json.dumps(tools)}) + db.commit() + + +def set_agent_task_description( + db: Session, task_id: uuid.UUID, description: str +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"task_description": description}) + db.commit() + + +def set_agent_task_framework_version( + db: Session, task_id: uuid.UUID, framework_version: str +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"framework_version": framework_version}) + db.commit() + + +# ── Agent events ──────────────────────────────────────────────────────────── + + +def get_agent_events_by_task( + db: Session, + task_id: uuid.UUID, + event_type: Optional[str] = None, +) -> List[db_schemas.AgentEvent]: + q = db.query(db_schemas.AgentEvent).filter(db_schemas.AgentEvent.task_id == task_id) + if event_type is not None: + q = q.filter(db_schemas.AgentEvent.event_type == event_type) + return q.order_by(db_schemas.AgentEvent.event_index).all() + + +def get_last_agent_event_for_session( + db: Session, + session_id: uuid.UUID, + event_type: Optional[str] = None, +) -> Optional[db_schemas.AgentEvent]: + """Most recent event across all tasks belonging to this session — used to detect + chat-session boundaries that span idle-close task gaps.""" + q = ( + db.query(db_schemas.AgentEvent) + .join( + db_schemas.AgentTask, + db_schemas.AgentEvent.task_id == db_schemas.AgentTask.task_id, + ) + .filter(db_schemas.AgentTask.session_id == session_id) + ) + if event_type is not None: + q = q.filter(db_schemas.AgentEvent.event_type == event_type) + return q.order_by(db_schemas.AgentEvent.created_at.desc()).first() + + +def count_agent_events_for_task(db: Session, task_id: uuid.UUID) -> int: + """Next free ``event_index`` for a task (events are 0-indexed and append-only).""" + return ( + db.query(func.count(db_schemas.AgentEvent.event_id)) + .filter(db_schemas.AgentEvent.task_id == task_id) + .scalar() + or 0 + ) + + +def find_existing_agent_event_span_ids( + db: Session, span_ids: List[str] +) -> List[str]: + """Which of ``span_ids`` have already been ingested for this task set. + + Used to make self-report ingestion idempotent: a retried batch is detected + rather than duplicated. + """ + if not span_ids: + return [] + return [ + row.span_id + for row in db.query(db_schemas.AgentEvent.span_id) + .filter(db_schemas.AgentEvent.span_id.in_(span_ids)) + .all() + ] + + +def append_agent_event( + db: Session, + task_id: uuid.UUID, + event_index: int, + event_type: str, + latency_ms: Optional[int] = None, + source: Optional[str] = None, + schema_version: Optional[str] = None, + occurred_at: Optional[datetime] = None, + # Shared span identifiers (model_call and tool_call) + span_id: Optional[str] = None, + parent_span_id: Optional[uuid.UUID] = None, + request_id: Optional[str] = None, + chat_session_index: Optional[int] = None, + # model_call fields + model: Optional[str] = None, + agent_profile: Optional[str] = None, + streaming: Optional[bool] = None, + message_count: Optional[int] = None, + role_system_count: Optional[int] = None, + role_user_count: Optional[int] = None, + role_assistant_count: Optional[int] = None, + role_tool_count: Optional[int] = None, + tools_kept: Optional[int] = None, + tools_stripped: Optional[int] = None, + tool_names_requested: Optional[list] = None, + max_tokens: Optional[int] = None, + prompt_tokens: Optional[int] = None, + completion_tokens: Optional[int] = None, + total_tokens: Optional[int] = None, + finish_reason: Optional[str] = None, + upstream_status: Optional[int] = None, + step_index: Optional[int] = None, + context_window_size_bytes: Optional[int] = None, + active_file: Optional[str] = None, + first_message_hash: Optional[str] = None, + chat_new_session_detected: Optional[bool] = None, + experiment_tool_access_enabled: Optional[bool] = None, + experiment_approval_policy: Optional[str] = None, + # model_call content (only when store_agent_content resolves True) + first_system_message: Optional[str] = None, + last_user_message: Optional[str] = None, + response_text: Optional[str] = None, + # tool_call fields + tool_name: Optional[str] = None, + tool_arguments_length: Optional[int] = None, + tool_result_length: Optional[int] = None, + # tool_call content (only when store_agent_content resolves True) + tool_arguments: Optional[str] = None, + tool_result: Optional[str] = None, + # JSON overflow + extra_json: Optional[str] = None, + # content (only when store_agent_content resolves True) + payload_json: Optional[str] = None, + commit: bool = True, +) -> db_schemas.AgentEvent: + """Append one event row. + + ``commit=False`` lets a caller stage a whole batch and commit once (used by + the self-report ingestion path, where a partially-written batch would leave + gaps in ``event_index``). + """ + event = db_schemas.AgentEvent( + event_id=uuid.uuid4(), + task_id=task_id, + event_index=event_index, + event_type=event_type, + source=source, + schema_version=schema_version, + latency_ms=latency_ms, + occurred_at=occurred_at, + span_id=span_id, + parent_span_id=parent_span_id, + request_id=request_id, + chat_session_index=chat_session_index, + model=model, + agent_profile=agent_profile, + streaming=streaming, + message_count=message_count, + role_system_count=role_system_count, + role_user_count=role_user_count, + role_assistant_count=role_assistant_count, + role_tool_count=role_tool_count, + tools_kept=tools_kept, + tools_stripped=tools_stripped, + tool_names_requested=tool_names_requested, + max_tokens=max_tokens, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + finish_reason=finish_reason, + upstream_status=upstream_status, + step_index=step_index, + context_window_size_bytes=context_window_size_bytes, + active_file=active_file, + first_message_hash=first_message_hash, + chat_new_session_detected=chat_new_session_detected, + experiment_tool_access_enabled=experiment_tool_access_enabled, + experiment_approval_policy=experiment_approval_policy, + first_system_message=first_system_message, + last_user_message=last_user_message, + response_text=response_text, + tool_name=tool_name, + tool_arguments_length=tool_arguments_length, + tool_result_length=tool_result_length, + tool_arguments=tool_arguments, + tool_result=tool_result, + extra_json=extra_json, + payload_json=payload_json, + ) + db.add(event) + if commit: + db.commit() + db.refresh(event) + else: + db.flush() + return event + + +# ── Agent edits (human-in-the-loop decisions) ─────────────────────────────── + + +def create_agent_edit( + db: Session, + task_id: uuid.UUID, + file_path: str, + diff_text: Optional[str] = None, +) -> db_schemas.AgentEdit: + edit = db_schemas.AgentEdit( + edit_id=uuid.uuid4(), + task_id=task_id, + file_path=file_path, + diff_text=diff_text, + ) + db.add(edit) + db.commit() + db.refresh(edit) + return edit + + +def get_agent_edits_by_task( + db: Session, task_id: uuid.UUID +) -> List[db_schemas.AgentEdit]: + return ( + db.query(db_schemas.AgentEdit) + .filter(db_schemas.AgentEdit.task_id == task_id) + .all() + ) + + +def update_agent_edit_decision( + db: Session, + edit_id: uuid.UUID, + was_accepted: bool, + was_modified: Optional[bool] = None, + edit_delta_json: Optional[str] = None, +) -> bool: + data: dict = {"was_accepted": was_accepted, "decided_at": datetime.now()} + if was_modified is not None: + data["was_modified"] = was_modified + if edit_delta_json is not None: + data["edit_delta_json"] = edit_delta_json + result = ( + db.query(db_schemas.AgentEdit) + .filter(db_schemas.AgentEdit.edit_id == edit_id) + .update(data) + ) + db.commit() + return result > 0 + + +# ── Agent memory (survives IDE restarts) ──────────────────────────────────── + + +def get_agent_memory_by_session_id( + db: Session, session_id: str +) -> Optional[db_schemas.AgentMemory]: + return ( + db.query(db_schemas.AgentMemory) + .filter(db_schemas.AgentMemory.session_id == session_id) + .first() + ) + + +def upsert_agent_memory( + db: Session, + *, + session_id: str, + owner_user_id: str, + owner_project_id: str, + messages: List[dict], +) -> db_schemas.AgentMemory: + """Replace the stored memory snapshot for one agent session. + + Ownership is written from the server-derived ACP scope, never from the + request body, so a client cannot attach its memory to someone else's + session. + """ + memory_json = json.dumps({"messages": messages}, sort_keys=True) + existing_memory = get_agent_memory_by_session_id(db, session_id) + now = datetime.now() + if existing_memory is None: + agent_memory = db_schemas.AgentMemory( + session_id=session_id, + owner_user_id=owner_user_id, + owner_project_id=owner_project_id, + memory_json=memory_json, + created_at=now, + updated_at=now, + ) + db.add(agent_memory) + else: + existing_memory.owner_user_id = owner_user_id + existing_memory.owner_project_id = owner_project_id + existing_memory.memory_json = memory_json + existing_memory.updated_at = now + agent_memory = existing_memory + db.commit() + db.refresh(agent_memory) + return agent_memory + + +def delete_agent_memory(db: Session, session_id: str) -> bool: + result = ( + db.query(db_schemas.AgentMemory) + .filter(db_schemas.AgentMemory.session_id == session_id) + .delete() + ) + db.commit() + return result > 0 diff --git a/src/database/db_schemas.py b/src/database/db_schemas.py index 7d0ee69..7e4c00e 100644 --- a/src/database/db_schemas.py +++ b/src/database/db_schemas.py @@ -42,8 +42,25 @@ "store_context": False, # Whether to store code context "store_contextual_telemetry": True, # Whether to collect contextual data "store_behavioral_telemetry": True, # Whether to collect behavioral data + # Whether to store agent task/event/edit *content* (message text, tool + # arguments/results, diffs). Structural telemetry (tokens, latency, span + # tree, tool names) is always stored and is not covered by this flag. + # + # Default ON: this is a research platform where study participants give + # informed consent through a separate process before using it, so opt-out + # is the right default for research data collection. The *mechanism* stays + # strict — enforcement is server-side only (see + # backend.routers.agent.consent.resolve_store_agent_content); a + # client-supplied flag is never trusted. + "store_agent_content": True, } +# Preference key gating persistence of agent content columns. +STORE_AGENT_CONTENT_KEY = "store_agent_content" +# Server-side default applied when a user row predates this preference (i.e. the +# key is absent from their stored preference JSON). +STORE_AGENT_CONTENT_DEFAULT = True + class Config(Base): """ @@ -693,6 +710,386 @@ class ConfigAssignmentHistory(Base): assigned_config = relationship("Config") +# ── Agent tables ────────────────────────────────────────────────────────────── +# +# One schema serves both agent runtimes: +# +# * the built-in ``code4me2-agent`` ReAct loop, which runs locally and +# self-reports its steps to POST /api/agent/events/ingest, and +# * third-party ACP agents (Goose, Codex), which are observed transparently +# by the plugin's local proxy relaying through POST /api/agent/inference. +# +# Shape: "columnar core + JSON overflow". Fields both runtimes produce get +# first-class typed columns (span tree, token counts, latency, tool names, +# finish reason); runtime-specific or experimental extras go into the single +# ``extra_json`` overflow column so adding an adapter never needs a migration. +# +# Privacy split: structural columns (status, timings, token counts, decisions) +# are always stored. Content columns (task_description, first_system_message, +# last_user_message, response_text, tool_arguments, tool_result, payload_json, +# diff_text, edit_delta_json) are nullable and only ever written when the +# requesting user's ``store_agent_content`` preference resolves True — checked +# server-side, never taken from a client-supplied flag. + + +class AgentProfile(Base): + """Named experiment variant — defines runtime, provider, tools and policy. + + ``framework_version`` selects which agent runtime the profile targets + (``code4me2-agent``, ``goose``, ``codex``), and therefore which telemetry + path a task created from it will use. + + The provider triple (``base_url``, ``api_key_ref``, ``model``) is + deliberately generic: any endpoint speaking the OpenAI-compatible + chat-completions wire format works, so a local Ollama install, Groq, + OpenRouter and OpenAI itself are all a config change rather than a code + change. ``base_url`` NULL falls back to the server's configured default + (see ``agents.provider.resolve_upstream``). + """ + + __tablename__ = "agent_profile" + __table_args__ = ( + Index("idx_agent_profile_is_active", "is_active"), + {"schema": "public"}, + ) + + profile_id = Column(UUID(as_uuid=True), primary_key=True) + name = Column(String, unique=True, nullable=False) + model = Column(String, nullable=False) + # Agent runtime this profile targets: code4me2-agent | goose | codex + framework_version = Column( + String, nullable=False, server_default="code4me2-agent" + ) + # OpenAI-compatible base URL, e.g. http://localhost:11434/v1 (Ollama), + # https://api.groq.com/openai/v1, https://api.openai.com/v1. NULL = server default. + base_url = Column(String, nullable=True) + # Name of the environment variable holding the upstream API key — never the + # key itself, so profiles stay safe to dump from the admin UI or the DB. + api_key_ref = Column(String, nullable=True) + tools_json = Column(Text, nullable=False) # JSON array of tool names + approval_policy = Column(String, nullable=False) # suggestion_only | per_step | … + max_steps = Column(Integer, nullable=False) + # Sampling temperature injected into the upstream request, server-side, the same + # way `model` is overridden. NULL = don't inject; let the provider use its default. + temperature = Column(Double, nullable=True) + max_context_tokens = Column( + Integer, nullable=True + ) # per-turn rolling window; NULL = model max + # Only active profiles are candidate arms for new A/B assignments. Inactive + # profiles stay in the table (drafts, or arms retired mid-study) and existing + # users keep any assignment already pinned to them. + is_active = Column(Boolean, server_default="true", default=True, nullable=False) + created_at = Column(DateTime, default=datetime.now) + + +class AgentProfileAssignment(Base): + """Sticky user→profile mapping for A/B testing. + + Resolution is server-authoritative: on a user's first agent task we draw a + random active profile and persist it here, so the user keeps the same arm for + the life of the experiment regardless of later changes to the active set. + `source` distinguishes randomized assignments ("auto") from admin overrides + ("manual"); manual rows are excluded from re-rolls and from A/B analysis. + """ + + __tablename__ = "agent_profile_assignment" + __table_args__ = ( + Index("idx_agent_profile_assignment_profile_id", "profile_id"), + {"schema": "public"}, + ) + + user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + primary_key=True, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="CASCADE"), + nullable=False, + ) + source = Column(String, nullable=False, server_default="auto") # auto | manual + assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + # Relationships + user = relationship("User") + profile = relationship("AgentProfile") + + +class AgentTask(Base): + """One row per agent session a developer starts. + + A "session" here is one run of whichever agent runtime the assigned profile + selects — Goose, Codex, or the built-in ``code4me2-agent`` ReAct loop. + + The profile's config (model, temperature, approval_policy, tools_json) is + *snapshotted* onto this row at creation time, so later edits to the profile + never retroactively change the conditions a completed task ran under. + """ + + __tablename__ = "agent_task" + __table_args__ = ( + Index("idx_agent_task_status", "status"), + Index("idx_agent_task_session_id", "session_id"), + Index("idx_agent_task_owner_user_id", "owner_user_id"), + {"schema": "public"}, + ) + + task_id = Column(UUID(as_uuid=True), primary_key=True) + # nullable — populated when the task is minted via the authenticated plugin endpoint + session_id = Column( + UUID(as_uuid=True), ForeignKey("public.session.session_id"), nullable=True + ) + # Which path created this task: "plugin" (proxy relay, third-party agents), + # "code4me2_agent" (self-reporting built-in runtime), or "benchmark". + source = Column(String, nullable=False, server_default="plugin") + # Owner attribution for tasks that arrive over the ACP bearer path, where + # there is no session cookie to resolve the user from. + owner_user_id = Column( + UUID(as_uuid=True), ForeignKey("public.user.user_id"), nullable=True + ) + owner_project_id = Column(UUID(as_uuid=True), nullable=True) + # The runtime's *own* identifiers, which are not guaranteed to be UUIDs + # (code4me2-agent uses uuid4().hex; ACP session ids are opaque strings). + # Kept alongside the server-side UUID PK so self-reported batches can be + # matched back to their task without forcing the runtime to adopt our ids. + external_run_id = Column(String, unique=True, nullable=True) + agent_session_id = Column(String, nullable=True) + agent_profile = Column(String, nullable=False) + model = Column(String, nullable=False) + # Snapshotted from the assigned profile at task-creation time, like `model`, so + # the value is stable for the life of the task. NULL = provider default. + temperature = Column(Double, nullable=True) + approval_policy = Column(String, nullable=False) + tools_json = Column(Text, nullable=False) + # Agent runtime (e.g. "goose", "codex", "code4me2-agent") — snapshotted from + # the profile, but left nullable so the proxy can backfill the concrete + # version string it observes ("goose 1.x") on the first inference call. + framework_version = Column(String, nullable=True) + status = Column(String, nullable=False) # pending | running | done | failed + created_at = Column(DateTime, default=datetime.now) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime, nullable=True) + total_steps = Column(Integer, nullable=True) + input_tokens = Column(Integer, nullable=True) + output_tokens = Column(Integer, nullable=True) + # Points at the most recent event, so a dashboard can show "where is this + # task now" without scanning agent_event. + # + # agent_task and agent_event reference each other, which is a deliberate + # cycle. `use_alter=True` tells SQLAlchemy to emit this constraint as a + # separate ALTER after both tables exist, so metadata.create_all() can + # order the DDL instead of erroring on the cycle. The Alembic migration + # does the same thing by hand. + latest_event_id = Column( + UUID(as_uuid=True), + ForeignKey( + "public.agent_event.event_id", + use_alter=True, + name="agent_task_latest_event_id_fkey", + ), + nullable=True, + ) + # Content — written only when store_agent_content resolves True + task_description = Column(Text, nullable=True) + + +class AgentEvent(Base): + """One row per agent step, from either telemetry path. + + ``event_type`` is the unified vocabulary both paths emit into: + ``model_call`` and ``tool_call`` are the two that carry typed columns; + ``thought`` / ``observation`` / ``run_started`` / ``run_completed`` / + ``decision`` are structural markers. + + Rows form a span tree via ``span_id`` / ``parent_span_id`` (tool calls nest + under the model_call whose response requested them). Anything a specific + adapter reports that has no typed column lands in ``extra_json``. + """ + + __tablename__ = "agent_event" + __table_args__ = ( + Index("idx_agent_event_task_id", "task_id"), + Index("idx_agent_event_event_type", "event_type"), + Index("idx_agent_event_created_at", "created_at"), + {"schema": "public"}, + ) + + event_id = Column(UUID(as_uuid=True), primary_key=True) + task_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_task.task_id", ondelete="CASCADE"), + nullable=False, + ) + event_index = Column(Integer, nullable=False) + # model_call | tool_call | thought | observation | test_run | decision | + # run_started | run_completed | accepted | rejected + event_type = Column(String, nullable=False) + # Which telemetry path wrote this row: "proxy" (client-side relay observing + # a third-party agent) or "code4me2_agent" (runtime self-report). + source = Column(String, nullable=True) + # Envelope schema version reported by a self-reporting runtime, so old + # batches stay interpretable after the envelope evolves. + schema_version = Column(String, nullable=True) + latency_ms = Column(Integer, nullable=True) + created_at = Column(DateTime, default=datetime.now) + # Client-side event timestamp, as reported by a self-reporting runtime. + # Distinct from created_at, which is when the server persisted the row. + occurred_at = Column(DateTime(timezone=True), nullable=True) + + # Shared span identifiers (model_call and tool_call). + # Events are grouped by task via task_id; there is no separate trace_id column — + # it was always a duplicate of task_id. + span_id = Column(String, nullable=True) + parent_span_id = Column(UUID(as_uuid=True), nullable=True) + # Correlates every event emitted while serving one user prompt. + request_id = Column(String, nullable=True) + chat_session_index = Column(Integer, nullable=True) + + # ── model_call fields — null for tool_call rows ── + model = Column(String, nullable=True) + agent_profile = Column(String, nullable=True) + streaming = Column(Boolean, nullable=True) + message_count = Column(Integer, nullable=True) + # Per-role message counts (the shape of the context window). + role_system_count = Column(Integer, nullable=True) + role_user_count = Column(Integer, nullable=True) + role_assistant_count = Column(Integer, nullable=True) + role_tool_count = Column(Integer, nullable=True) + tools_kept = Column(Integer, nullable=True) + tools_stripped = Column(Integer, nullable=True) + tool_names_requested = Column(ARRAY(String), nullable=True) + max_tokens = Column(Integer, nullable=True) + prompt_tokens = Column(Integer, nullable=True) + completion_tokens = Column(Integer, nullable=True) + total_tokens = Column(Integer, nullable=True) + finish_reason = Column(String, nullable=True) + upstream_status = Column(Integer, nullable=True) + step_index = Column(Integer, nullable=True) + context_window_size_bytes = Column(Integer, nullable=True) + active_file = Column(Text, nullable=True) + first_message_hash = Column(String, nullable=True) + chat_new_session_detected = Column(Boolean, nullable=True) + experiment_tool_access_enabled = Column(Boolean, nullable=True) + experiment_approval_policy = Column(String, nullable=True) + # Content — written only when store_agent_content resolves True + first_system_message = Column(Text, nullable=True) + last_user_message = Column(Text, nullable=True) + response_text = Column(Text, nullable=True) + + # ── tool_call fields — null for model_call rows ── + tool_name = Column(String, nullable=True) + tool_arguments_length = Column(Integer, nullable=True) + tool_result_length = Column(Integer, nullable=True) + # Content — written only when store_agent_content resolves True + tool_arguments = Column(Text, nullable=True) + tool_result = Column(Text, nullable=True) + + # ── JSON overflow ── + # Adapter-specific / experimental structural metrics that don't warrant a + # typed column (e.g. the ReAct adapter's per-iteration counters, an + # adapter's own observability block). Never content — see payload_json. + extra_json = Column(Text, nullable=True) + # Free-form event payload from a self-reporting runtime. May quote user + # prompts, model output, or file contents, so this is Content — written only + # when store_agent_content resolves True. + payload_json = Column(Text, nullable=True) + + +class AgentEdit(Base): + """One row per file the agent proposed changing. + + The accept / reject / modified decision plus the diff is the + human-in-the-loop signal: it records not just what the agent suggested but + what the developer actually did with it. + """ + + __tablename__ = "agent_edit" + __table_args__ = ( + Index("idx_agent_edit_task_id", "task_id"), + {"schema": "public"}, + ) + + edit_id = Column(UUID(as_uuid=True), primary_key=True) + task_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_task.task_id", ondelete="CASCADE"), + nullable=False, + ) + file_path = Column(Text, nullable=False) + was_accepted = Column(Boolean, nullable=True) # True/False after developer decides + was_modified = Column( + Boolean, nullable=True + ) # True if developer edited before accepting + decided_at = Column(DateTime, nullable=True) + # Content — written only when store_agent_content resolves True + diff_text = Column(Text, nullable=True) + edit_delta_json = Column( + Text, nullable=True + ) # what developer changed before accepting + + +class AgentMemory(Base): + """Durable per-session agent memory snapshot. + + Holds the latest serialized memory window for one agent chat session, so + conversation state survives an IDE (or agent process) restart. Ownership is + server-derived from the ACP token, matching agent telemetry authorization — + the client cannot claim a session it doesn't own. + """ + + __tablename__ = "agent_memory" + __table_args__ = ( + Index("idx_agent_memory_owner_user_id", "owner_user_id"), + Index("idx_agent_memory_owner_project_id", "owner_project_id"), + Index("idx_agent_memory_updated_at", "updated_at"), + {"schema": "public"}, + ) + + # ACP session ids are opaque strings, not guaranteed to be UUIDs. + session_id = Column(String, primary_key=True) + owner_user_id = Column(String, nullable=False) + owner_project_id = Column(String, nullable=False) + memory_json = Column(Text, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + updated_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class StudyAgentProfile(Base): + """Links a study to the agent profiles that serve as its experiment arms. + + A study with rows here drives agent-profile A/B assignment: its selected + profiles become the candidate pool for new auto-draws (see + ``registry.resolve_assignment``). Studies with no rows here are + completion-only and behave exactly as before this table existed. + + ``is_baseline`` marks the arm that agent-evaluation uplift is measured + against (analogous to ``study.default_config_id`` on the completion side). + """ + + __tablename__ = "study_agent_profile" + __table_args__ = ( + Index("idx_study_agent_profile_profile_id", "profile_id"), + {"schema": "public"}, + ) + + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + primary_key=True, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id"), + primary_key=True, + ) + is_baseline = Column(Boolean, server_default="false", default=False, nullable=False) + + # Relationships + study = relationship("Study") + profile = relationship("AgentProfile") + + # # class SessionQuery(Base): # __tablename__ = "session_queries" diff --git a/src/database/migration/migration_manager.py b/src/database/migration/migration_manager.py index 561c94f..5495918 100644 --- a/src/database/migration/migration_manager.py +++ b/src/database/migration/migration_manager.py @@ -142,24 +142,21 @@ def initialize_from_sql(self) -> bool: return False def setup_migration_tracking(self) -> bool: - """Set up Alembic migration tracking.""" + """Set up Alembic tracking by stamping at the start of the existing chain.""" print("Setting up migration tracking...") try: - # Set database URL self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) - # Create EMPTY baseline migration (not autogenerated) - # This represents the current state from init.sql - init_file_name = "init_test.sql" if self.use_test_db else "init.sql" - command.revision( - self.alembic_cfg, - message=f"Baseline migration from {init_file_name}", - autogenerate=False, # Create empty migration - ) - - # Mark as current state - command.stamp(self.alembic_cfg, "head") + # Stamp at "base" (no revision) so the subsequent `upgrade head` runs + # every migration in order from the committed baseline + # (c2dc3e9cc1bc_baseline_migration_from_init_sql). + # + # Do NOT create a new revision file here: the baseline is already + # committed, so generating another one would put this database on a + # parallel dead-end branch and silently skip all real migrations — + # which is how the agent tables would go missing on a fresh install. + command.stamp(self.alembic_cfg, "base") print("Migration tracking set up") return True diff --git a/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py b/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py new file mode 100644 index 0000000..db35a66 --- /dev/null +++ b/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py @@ -0,0 +1,27 @@ +"""Baseline migration from init.sql + +Deliberately empty. A fresh database is created by executing ``init.sql`` +(see ``migration_manager.initialize_from_sql``); this revision exists only to +give that state a name so subsequent migrations have something to chain from. + +Revision ID: c2dc3e9cc1bc +Revises: +Create Date: 2026-09-06 + +""" + +# revision identifiers, used by Alembic. +revision = "c2dc3e9cc1bc" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade database schema.""" + pass + + +def downgrade() -> None: + """Downgrade database schema.""" + pass diff --git a/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py b/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py new file mode 100644 index 0000000..ce02453 --- /dev/null +++ b/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py @@ -0,0 +1,396 @@ +"""add unified agent tables + +Creates the seven tables backing the agent subsystem in their final, +already-unified shape: + + agent_profile experiment variant: runtime + provider + tools + policy + agent_profile_assignment sticky per-user A/B arm + agent_task one row per agent session, config snapshotted + agent_event one row per step, "columnar core + JSON overflow" + agent_edit per-file proposal + accept/reject/modified + diff + agent_memory session-scoped memory surviving IDE restarts + study_agent_profile links the existing Study A/B framework to agent arms + +``agent_event`` gives typed columns to everything both telemetry paths produce +(the client-side proxy observing Goose/Codex, and the built-in code4me2-agent +runtime self-reporting), and routes adapter-specific extras into a single +``extra_json`` overflow column so a new adapter never requires a migration. + +Content columns (task_description, first_system_message, last_user_message, +response_text, tool_arguments, tool_result, payload_json, diff_text, +edit_delta_json) are all nullable: they are only ever written when the user's +``store_agent_content`` preference resolves True, enforced server-side. + +Revision ID: d1a2b3c4e5f7 +Revises: c2dc3e9cc1bc +Create Date: 2026-09-06 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "d1a2b3c4e5f7" +down_revision = "c2dc3e9cc1bc" +branch_labels = None +depends_on = None + +_SCHEMA = "public" + + +def upgrade() -> None: + op.create_table( + "agent_profile", + sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("model", sa.String(), nullable=False), + # Which agent runtime this profile targets: code4me2-agent | goose | codex + sa.Column( + "framework_version", + sa.String(), + nullable=False, + server_default=sa.text("'code4me2-agent'"), + ), + # Generic OpenAI-compatible provider triple. NULL base_url = server default. + sa.Column("base_url", sa.String(), nullable=True), + # Name of the env var holding the key — never the key itself. + sa.Column("api_key_ref", sa.String(), nullable=True), + sa.Column("tools_json", sa.Text(), nullable=False), + sa.Column("approval_policy", sa.String(), nullable=False), + sa.Column("max_steps", sa.Integer(), nullable=False), + sa.Column("temperature", sa.Double(), nullable=True), + sa.Column("max_context_tokens", sa.Integer(), nullable=True), + sa.Column( + "is_active", sa.Boolean(), nullable=False, server_default=sa.text("true") + ), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("profile_id"), + sa.UniqueConstraint("name"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_profile_is_active", + "agent_profile", + ["is_active"], + unique=False, + schema=_SCHEMA, + ) + + op.create_table( + "agent_profile_assignment", + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "source", sa.String(), nullable=False, server_default=sa.text("'auto'") + ), + sa.Column( + "assigned_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["user_id"], ["public.user.user_id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["profile_id"], + ["public.agent_profile.profile_id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("user_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_profile_assignment_profile_id", + "agent_profile_assignment", + ["profile_id"], + unique=False, + schema=_SCHEMA, + ) + + # agent_task and agent_event reference each other (agent_event.task_id → + # agent_task, agent_task.latest_event_id → agent_event), so the cycle is + # broken by adding latest_event_id's FK after both tables exist. + op.create_table( + "agent_task", + sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("session_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column( + "source", sa.String(), nullable=False, server_default=sa.text("'plugin'") + ), + sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("owner_project_id", postgresql.UUID(as_uuid=True), nullable=True), + # The runtime's own ids, which are not guaranteed to be UUID-shaped. + sa.Column("external_run_id", sa.String(), nullable=True), + sa.Column("agent_session_id", sa.String(), nullable=True), + sa.Column("agent_profile", sa.String(), nullable=False), + sa.Column("model", sa.String(), nullable=False), + sa.Column("temperature", sa.Double(), nullable=True), + sa.Column("approval_policy", sa.String(), nullable=False), + sa.Column("tools_json", sa.Text(), nullable=False), + sa.Column("framework_version", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(), nullable=True), + sa.Column("total_steps", sa.Integer(), nullable=True), + sa.Column("input_tokens", sa.Integer(), nullable=True), + sa.Column("output_tokens", sa.Integer(), nullable=True), + sa.Column("latest_event_id", postgresql.UUID(as_uuid=True), nullable=True), + # Content — gated by store_agent_content + sa.Column("task_description", sa.Text(), nullable=True), + sa.ForeignKeyConstraint(["session_id"], ["public.session.session_id"]), + sa.ForeignKeyConstraint(["owner_user_id"], ["public.user.user_id"]), + sa.PrimaryKeyConstraint("task_id"), + sa.UniqueConstraint("external_run_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_task_status", "agent_task", ["status"], unique=False, schema=_SCHEMA + ) + op.create_index( + "idx_agent_task_session_id", + "agent_task", + ["session_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_task_owner_user_id", + "agent_task", + ["owner_user_id"], + unique=False, + schema=_SCHEMA, + ) + + op.create_table( + "agent_event", + sa.Column("event_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("event_index", sa.Integer(), nullable=False), + sa.Column("event_type", sa.String(), nullable=False), + # "proxy" | "code4me2_agent" + sa.Column("source", sa.String(), nullable=True), + sa.Column("schema_version", sa.String(), nullable=True), + sa.Column("latency_ms", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=True), + # Span tree + sa.Column("span_id", sa.String(), nullable=True), + sa.Column("parent_span_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("request_id", sa.String(), nullable=True), + sa.Column("chat_session_index", sa.Integer(), nullable=True), + # model_call fields + sa.Column("model", sa.String(), nullable=True), + sa.Column("agent_profile", sa.String(), nullable=True), + sa.Column("streaming", sa.Boolean(), nullable=True), + sa.Column("message_count", sa.Integer(), nullable=True), + sa.Column("role_system_count", sa.Integer(), nullable=True), + sa.Column("role_user_count", sa.Integer(), nullable=True), + sa.Column("role_assistant_count", sa.Integer(), nullable=True), + sa.Column("role_tool_count", sa.Integer(), nullable=True), + sa.Column("tools_kept", sa.Integer(), nullable=True), + sa.Column("tools_stripped", sa.Integer(), nullable=True), + sa.Column("tool_names_requested", postgresql.ARRAY(sa.String()), nullable=True), + sa.Column("max_tokens", sa.Integer(), nullable=True), + sa.Column("prompt_tokens", sa.Integer(), nullable=True), + sa.Column("completion_tokens", sa.Integer(), nullable=True), + sa.Column("total_tokens", sa.Integer(), nullable=True), + sa.Column("finish_reason", sa.String(), nullable=True), + sa.Column("upstream_status", sa.Integer(), nullable=True), + sa.Column("step_index", sa.Integer(), nullable=True), + sa.Column("context_window_size_bytes", sa.Integer(), nullable=True), + sa.Column("active_file", sa.Text(), nullable=True), + sa.Column("first_message_hash", sa.String(), nullable=True), + sa.Column("chat_new_session_detected", sa.Boolean(), nullable=True), + sa.Column("experiment_tool_access_enabled", sa.Boolean(), nullable=True), + sa.Column("experiment_approval_policy", sa.String(), nullable=True), + # model_call content — gated by store_agent_content + sa.Column("first_system_message", sa.Text(), nullable=True), + sa.Column("last_user_message", sa.Text(), nullable=True), + sa.Column("response_text", sa.Text(), nullable=True), + # tool_call fields + sa.Column("tool_name", sa.String(), nullable=True), + sa.Column("tool_arguments_length", sa.Integer(), nullable=True), + sa.Column("tool_result_length", sa.Integer(), nullable=True), + # tool_call content — gated by store_agent_content + sa.Column("tool_arguments", sa.Text(), nullable=True), + sa.Column("tool_result", sa.Text(), nullable=True), + # JSON overflow: adapter-specific structural metrics (never content) + sa.Column("extra_json", sa.Text(), nullable=True), + # Free-form runtime payload — content, gated by store_agent_content + sa.Column("payload_json", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["task_id"], ["public.agent_task.task_id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("event_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_event_task_id", + "agent_event", + ["task_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_event_event_type", + "agent_event", + ["event_type"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_event_created_at", + "agent_event", + ["created_at"], + unique=False, + schema=_SCHEMA, + ) + op.create_foreign_key( + "agent_task_latest_event_id_fkey", + "agent_task", + "agent_event", + ["latest_event_id"], + ["event_id"], + source_schema=_SCHEMA, + referent_schema=_SCHEMA, + ) + + op.create_table( + "agent_edit", + sa.Column("edit_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("file_path", sa.Text(), nullable=False), + sa.Column("was_accepted", sa.Boolean(), nullable=True), + sa.Column("was_modified", sa.Boolean(), nullable=True), + sa.Column("decided_at", sa.DateTime(), nullable=True), + # Content — gated by store_agent_content + sa.Column("diff_text", sa.Text(), nullable=True), + sa.Column("edit_delta_json", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["task_id"], ["public.agent_task.task_id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("edit_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_edit_task_id", + "agent_edit", + ["task_id"], + unique=False, + schema=_SCHEMA, + ) + + op.create_table( + "agent_memory", + # ACP session ids are opaque strings, not guaranteed to be UUIDs. + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("owner_user_id", sa.String(), nullable=False), + sa.Column("owner_project_id", sa.String(), nullable=False), + sa.Column("memory_json", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("session_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_memory_owner_user_id", + "agent_memory", + ["owner_user_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_memory_owner_project_id", + "agent_memory", + ["owner_project_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_memory_updated_at", + "agent_memory", + ["updated_at"], + unique=False, + schema=_SCHEMA, + ) + + op.create_table( + "study_agent_profile", + sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "is_baseline", sa.Boolean(), nullable=False, server_default=sa.text("false") + ), + sa.ForeignKeyConstraint( + ["study_id"], ["public.study.study_id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["profile_id"], ["public.agent_profile.profile_id"]), + sa.PrimaryKeyConstraint("study_id", "profile_id"), + schema=_SCHEMA, + ) + op.create_index( + "idx_study_agent_profile_profile_id", + "study_agent_profile", + ["profile_id"], + unique=False, + schema=_SCHEMA, + ) + + +def downgrade() -> None: + op.drop_index( + "idx_study_agent_profile_profile_id", + table_name="study_agent_profile", + schema=_SCHEMA, + ) + op.drop_table("study_agent_profile", schema=_SCHEMA) + + for index in ( + "idx_agent_memory_updated_at", + "idx_agent_memory_owner_project_id", + "idx_agent_memory_owner_user_id", + ): + op.drop_index(index, table_name="agent_memory", schema=_SCHEMA) + op.drop_table("agent_memory", schema=_SCHEMA) + + op.drop_index("idx_agent_edit_task_id", table_name="agent_edit", schema=_SCHEMA) + op.drop_table("agent_edit", schema=_SCHEMA) + + # Drop the cyclic FK before agent_event, so agent_event is droppable. + op.drop_constraint( + "agent_task_latest_event_id_fkey", + "agent_task", + type_="foreignkey", + schema=_SCHEMA, + ) + for index in ( + "idx_agent_event_created_at", + "idx_agent_event_event_type", + "idx_agent_event_task_id", + ): + op.drop_index(index, table_name="agent_event", schema=_SCHEMA) + op.drop_table("agent_event", schema=_SCHEMA) + + for index in ( + "idx_agent_task_owner_user_id", + "idx_agent_task_session_id", + "idx_agent_task_status", + ): + op.drop_index(index, table_name="agent_task", schema=_SCHEMA) + op.drop_table("agent_task", schema=_SCHEMA) + + op.drop_index( + "idx_agent_profile_assignment_profile_id", + table_name="agent_profile_assignment", + schema=_SCHEMA, + ) + op.drop_table("agent_profile_assignment", schema=_SCHEMA) + + op.drop_index( + "idx_agent_profile_is_active", table_name="agent_profile", schema=_SCHEMA + ) + op.drop_table("agent_profile", schema=_SCHEMA) diff --git a/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py b/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py new file mode 100644 index 0000000..fdcc32d --- /dev/null +++ b/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py @@ -0,0 +1,116 @@ +"""seed default agent profiles + +Without at least one *active* profile, POST /api/agent/task returns 503 ("No +active agent profiles are configured") and a freshly-migrated database cannot +mint tasks at all. This seeds one profile per supported runtime so the system +is usable out of the box. + +Only the built-in ``code4me2-agent`` profile is seeded active, because it is the +one runtime that needs no external binary: it points at a local Ollama endpoint, +which costs nothing and needs no API key. The Goose and Codex profiles are +seeded *inactive* — they require a Goose binary / Node toolchain on the +developer's machine, so an operator flips them on from the admin UI once those +prerequisites exist. Inactive profiles are never drawn as A/B arms. + +No API keys are stored here (or anywhere in the DB): ``api_key_ref`` names the +environment variable to read the key from at request time. + +Revision ID: e2b3c4d5f6a8 +Revises: d1a2b3c4e5f7 +Create Date: 2026-09-06 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e2b3c4d5f6a8" +down_revision = "d1a2b3c4e5f7" +branch_labels = None +depends_on = None + +# Fixed ids so the seeded profiles are identical across all environments. +_PROFILES = [ + { + "profile_id": "7e1f20ce-fc5c-4ba2-b908-eaf77047d4b4", + "name": "default-code4me2-agent", + "framework_version": "code4me2-agent", + # Ollama speaks the OpenAI-compatible chat-completions format, so the + # generic provider path works against it unchanged. + "base_url": "http://localhost:11434/v1", + "api_key_ref": "OLLAMA_API_KEY", + "model": "qwen2.5-coder:7b", + "tools_json": ( + '["read_file", "write_file", "create_file", "replace_text", ' + '"list_files", "search_files", "run_command"]' + ), + "approval_policy": "per_step", + "max_steps": 8, + "is_active": "true", + }, + { + "profile_id": "2b9d7c41-5a30-4c8e-9f21-1d6b0c3a7e58", + "name": "default-goose", + "framework_version": "goose", + "base_url": None, + "api_key_ref": "AGENT_UPSTREAM_API_KEY", + "model": "openai/gpt-oss-120b", + # Empty allowlist = the proxy forwards no tools. Populate from + # GET /api/agent/available-tools once a run has shown which tools the + # installed Goose build actually advertises. + "tools_json": "[]", + "approval_policy": "auto", + "max_steps": 15, + "is_active": "false", + }, + { + "profile_id": "6c4a8e12-93bf-4d57-8a0e-7f2c5b1d94a3", + "name": "default-codex", + "framework_version": "codex", + "base_url": "https://api.openai.com/v1", + "api_key_ref": "OPENAI_API_KEY", + "model": "gpt-5.1-codex-mini", + # Codex manages its own tool schemas over the Responses API; the proxy + # passes them through rather than filtering against an allowlist. + "tools_json": "[]", + "approval_policy": "auto", + "max_steps": 15, + "is_active": "false", + }, +] + + +def upgrade() -> None: + """Seed one profile per runtime. + + Idempotent: each insert is skipped when a profile of that name already + exists (e.g. created via the management website or a manual dev seed), so it + never collides on the unique name or the primary key. + """ + for profile in _PROFILES: + base_url = ( + "NULL" if profile["base_url"] is None else f"'{profile['base_url']}'" + ) + op.execute( + f""" + INSERT INTO public.agent_profile + (profile_id, name, model, framework_version, base_url, api_key_ref, + tools_json, approval_policy, max_steps, temperature, + max_context_tokens, is_active, created_at) + SELECT '{profile["profile_id"]}', '{profile["name"]}', + '{profile["model"]}', '{profile["framework_version"]}', + {base_url}, '{profile["api_key_ref"]}', + '{profile["tools_json"]}', '{profile["approval_policy"]}', + {profile["max_steps"]}, NULL, NULL, {profile["is_active"]}, now() + WHERE NOT EXISTS ( + SELECT 1 FROM public.agent_profile WHERE name = '{profile["name"]}' + ); + """ + ) + + +def downgrade() -> None: + """Remove only the rows this migration could have inserted (by their fixed + ids), so profiles created by other means are left untouched.""" + ids = ", ".join(f"'{profile['profile_id']}'" for profile in _PROFILES) + op.execute(f"DELETE FROM public.agent_profile WHERE profile_id IN ({ids});") diff --git a/src/performance_and_analysis/EvaluationScript.py b/src/performance_and_analysis/EvaluationScript.py index b9befbc..e0ef6a0 100644 --- a/src/performance_and_analysis/EvaluationScript.py +++ b/src/performance_and_analysis/EvaluationScript.py @@ -1,223 +1,223 @@ -import requests -import time -import os -import statistics -import math -from datasets import load_dataset -import sqlite3 -import json -from datetime import datetime -import pandas as pd - -# --- Configuration --- -API_ENDPOINT = "http://localhost:8008/api/completion/request" -MODEL_IDS = [1] - -# --- Authentication Tokens --- -AUTH_TOKEN = os.environ.get('AUTH_TOKEN', 'efde6a22-2d6a-4ba3-8efa-889530dd70cc') -PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', '418c70dd-867f-4935-a5e0-ba2d7ad94f5b') -SESSION_TOKEN = os.environ.get('SESSION_TOKEN', 'c39799cc-9d25-4da5-b2b3-f67d46ae09d1') - -# --- Benchmark Settings --- -DB_NAME = "benchmark_results.db" -CSV_NAME = "benchmark_results.csv" -NUM_WARMUP_RUNS = 10 -NUM_BENCHMARK_RUNS = 1 -BENCHMARK_SUBSET_SIZE = 1600 - - -# --- Helper Functions --- - -def setup_database(db_name="benchmark_results.db"): - """Creates a SQLite database and a 'results' table with separate response fields.""" - conn = sqlite3.connect(db_name) - cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema - cursor.execute(''' - CREATE TABLE IF NOT EXISTS results ( - timestamp TEXT, - model_ids TEXT, - prefix TEXT, - suffix TEXT, - e2e_time_ms REAL, - error TEXT, - response_message TEXT, - meta_query_id TEXT, - completion_model_id INTEGER, - completion_model_name TEXT, - completion_text TEXT, - generation_time_ms INTEGER, - confidence REAL - ) - ''') - conn.commit() - conn.close() - print(f"Database '{db_name}' and new 'results' table are ready.") - - -def load_benchmark_data(subset_size=None): - """Loads the SantaCoder FIM dataset from Hugging Face.""" - print("Downloading and preparing the santacoder-fim-task dataset (all languages)...") - try: - dataset = load_dataset("bigcode/santacoder-fim-task")['train'] - if subset_size and subset_size < len(dataset): - print(f"Using a subset of {subset_size} samples.") - return dataset.select(range(subset_size)) - print(f"Using the full dataset with {len(dataset)} samples.") - return dataset - except Exception as e: - print(f"Error loading dataset: {e}") - return None - - -def prepare_request_payload(benchmark_item, model_ids): - """Formats a dataset item into the required JSON payload.""" - return { - "model_ids": model_ids, - "context": { - "prefix": benchmark_item['prompt'], - "suffix": benchmark_item['suffix'], - "file_name": "main.py", - "context_files": [] - }, - "store_context": True, - "contextual_telemetry": {"language_id": 45, "trigger_type_id": 1, "version_id": 1, - "document_char_length": len(benchmark_item['prompt']) + len(benchmark_item['suffix'])}, - "store_contextual_telemetry": True, - "behavioral_telemetry": {"time_since_last_completion": 5000, "typing_speed": 300, - "relative_document_position": 0.5}, - "store_behavioral_telemetry": True - } - - -def send_request_and_measure_time(session, payload, cookies): - """Sends a request and returns a dictionary with time, response, and error.""" - start_time = time.perf_counter() - response_json, error_msg = None, None - try: - response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=30) - response.raise_for_status() - response_json = response.json() - except requests.exceptions.RequestException as e: - error_msg = str(e) - print(f"Request failed: {error_msg}") - finally: - end_time = time.perf_counter() - - duration_ms = (end_time - start_time) * 1000 - - return { - "e2e_time_ms": duration_ms, - "response_json": response_json, - "error": error_msg - } - - -def analyze_results(timings): - """Calculates and prints key statistics from the collected timings.""" - if not timings: - print("No successful requests to analyze.") - return - n = len(timings) - mean_val = statistics.mean(timings) - median_val = statistics.median(timings) - stdev_val = statistics.stdev(timings) if n > 1 else 0 - p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val - p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val - ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 - ci_lower = mean_val - ci_margin - ci_upper = mean_val + ci_margin - - print("\n--- Performance Analysis ---") - print(f"Total Successful Requests: {n}") - print(f"Mean Response Time: {mean_val:.2f} ms") - print(f"Median Response Time (P50):{median_val:.2f} ms") - print(f"Standard Deviation: {stdev_val:.2f} ms") - print(f"95th Percentile (P95): {p95:.2f} ms") - print(f"99th Percentile (P99): {p99:.2f} ms") - print(f"95% Confidence Interval: [{ci_lower:.2f} ms, {ci_upper:.2f} ms]") - print("--------------------------\n") - - -# --- Main Execution --- -def main(): - """Main function to run the benchmark.""" - print("Starting End-to-End Response Time Benchmark...") - setup_database(DB_NAME) - - cookies = { - 'auth_token': AUTH_TOKEN, - 'project_token': PROJECT_TOKEN, - 'session_token': SESSION_TOKEN - } - - dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) - if not dataset: return - - with requests.Session() as session: - print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") - for i in range(NUM_WARMUP_RUNS): - payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) - send_request_and_measure_time(session, payload, cookies) - - all_results_data = [] - print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") - for i in range(NUM_BENCHMARK_RUNS): - print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") - for j, item in enumerate(dataset): - payload = prepare_request_payload(item, MODEL_IDS) - result = send_request_and_measure_time(session, payload, cookies) - - # --- New Response Parsing Logic --- - response_data = result['response_json'] - completion_data = {} - message = None - meta_query_id = None - - if response_data: - message = response_data.get('message') - data = response_data.get('data', {}) - meta_query_id = data.get('meta_query_id') - completions = data.get('completions', []) - if completions: - completion_data = completions[0] # Get the first completion - - log_entry = { - "timestamp": datetime.now().isoformat(), - "model_ids": json.dumps(payload['model_ids']), - "prefix": payload['context']['prefix'], - "suffix": payload['context']['suffix'], - "e2e_time_ms": result['e2e_time_ms'], - "error": result['error'], - "response_message": message, - "meta_query_id": meta_query_id, - "completion_model_id": completion_data.get('model_id'), - "completion_model_name": completion_data.get('model_name'), - "completion_text": completion_data.get('completion'), - "generation_time_ms": completion_data.get('generation_time'), - "confidence": completion_data.get('confidence') - } - all_results_data.append(log_entry) - - if (j + 1) % 25 == 0: - print(f" ...completed {j + 1}/{len(dataset)} requests.") - - if not all_results_data: - print("No data collected. Exiting.") - return - - df = pd.DataFrame(all_results_data) - - with sqlite3.connect(DB_NAME) as conn: - df.to_sql('results', conn, if_exists='append', index=False) - print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") - - df.to_csv(CSV_NAME, index=False) - print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") - - successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() - analyze_results(successful_timings) - - -if __name__ == "__main__": +import requests +import time +import os +import statistics +import math +from datasets import load_dataset +import sqlite3 +import json +from datetime import datetime +import pandas as pd + +# --- Configuration --- +API_ENDPOINT = "http://localhost:8008/api/completion/request" +MODEL_IDS = [1] + +# --- Authentication Tokens --- +AUTH_TOKEN = os.environ.get('AUTH_TOKEN', 'efde6a22-2d6a-4ba3-8efa-889530dd70cc') +PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', '418c70dd-867f-4935-a5e0-ba2d7ad94f5b') +SESSION_TOKEN = os.environ.get('SESSION_TOKEN', 'c39799cc-9d25-4da5-b2b3-f67d46ae09d1') + +# --- Benchmark Settings --- +DB_NAME = "benchmark_results.db" +CSV_NAME = "benchmark_results.csv" +NUM_WARMUP_RUNS = 10 +NUM_BENCHMARK_RUNS = 1 +BENCHMARK_SUBSET_SIZE = 1600 + + +# --- Helper Functions --- + +def setup_database(db_name="benchmark_results.db"): + """Creates a SQLite database and a 'results' table with separate response fields.""" + conn = sqlite3.connect(db_name) + cursor = conn.cursor() + cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema + cursor.execute(''' + CREATE TABLE IF NOT EXISTS results ( + timestamp TEXT, + model_ids TEXT, + prefix TEXT, + suffix TEXT, + e2e_time_ms REAL, + error TEXT, + response_message TEXT, + meta_query_id TEXT, + completion_model_id INTEGER, + completion_model_name TEXT, + completion_text TEXT, + generation_time_ms INTEGER, + confidence REAL + ) + ''') + conn.commit() + conn.close() + print(f"Database '{db_name}' and new 'results' table are ready.") + + +def load_benchmark_data(subset_size=None): + """Loads the SantaCoder FIM dataset from Hugging Face.""" + print("Downloading and preparing the santacoder-fim-task dataset (all languages)...") + try: + dataset = load_dataset("bigcode/santacoder-fim-task")['train'] + if subset_size and subset_size < len(dataset): + print(f"Using a subset of {subset_size} samples.") + return dataset.select(range(subset_size)) + print(f"Using the full dataset with {len(dataset)} samples.") + return dataset + except Exception as e: + print(f"Error loading dataset: {e}") + return None + + +def prepare_request_payload(benchmark_item, model_ids): + """Formats a dataset item into the required JSON payload.""" + return { + "model_ids": model_ids, + "context": { + "prefix": benchmark_item['prompt'], + "suffix": benchmark_item['suffix'], + "file_name": "main.py", + "context_files": [] + }, + "store_context": True, + "contextual_telemetry": {"language_id": 45, "trigger_type_id": 1, "version_id": 1, + "document_char_length": len(benchmark_item['prompt']) + len(benchmark_item['suffix'])}, + "store_contextual_telemetry": True, + "behavioral_telemetry": {"time_since_last_completion": 5000, "typing_speed": 300, + "relative_document_position": 0.5}, + "store_behavioral_telemetry": True + } + + +def send_request_and_measure_time(session, payload, cookies): + """Sends a request and returns a dictionary with time, response, and error.""" + start_time = time.perf_counter() + response_json, error_msg = None, None + try: + response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=30) + response.raise_for_status() + response_json = response.json() + except requests.exceptions.RequestException as e: + error_msg = str(e) + print(f"Request failed: {error_msg}") + finally: + end_time = time.perf_counter() + + duration_ms = (end_time - start_time) * 1000 + + return { + "e2e_time_ms": duration_ms, + "response_json": response_json, + "error": error_msg + } + + +def analyze_results(timings): + """Calculates and prints key statistics from the collected timings.""" + if not timings: + print("No successful requests to analyze.") + return + n = len(timings) + mean_val = statistics.mean(timings) + median_val = statistics.median(timings) + stdev_val = statistics.stdev(timings) if n > 1 else 0 + p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val + p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val + ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 + ci_lower = mean_val - ci_margin + ci_upper = mean_val + ci_margin + + print("\n--- Performance Analysis ---") + print(f"Total Successful Requests: {n}") + print(f"Mean Response Time: {mean_val:.2f} ms") + print(f"Median Response Time (P50):{median_val:.2f} ms") + print(f"Standard Deviation: {stdev_val:.2f} ms") + print(f"95th Percentile (P95): {p95:.2f} ms") + print(f"99th Percentile (P99): {p99:.2f} ms") + print(f"95% Confidence Interval: [{ci_lower:.2f} ms, {ci_upper:.2f} ms]") + print("--------------------------\n") + + +# --- Main Execution --- +def main(): + """Main function to run the benchmark.""" + print("Starting End-to-End Response Time Benchmark...") + setup_database(DB_NAME) + + cookies = { + 'auth_token': AUTH_TOKEN, + 'project_token': PROJECT_TOKEN, + 'session_token': SESSION_TOKEN + } + + dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) + if not dataset: return + + with requests.Session() as session: + print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") + for i in range(NUM_WARMUP_RUNS): + payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) + send_request_and_measure_time(session, payload, cookies) + + all_results_data = [] + print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") + for i in range(NUM_BENCHMARK_RUNS): + print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") + for j, item in enumerate(dataset): + payload = prepare_request_payload(item, MODEL_IDS) + result = send_request_and_measure_time(session, payload, cookies) + + # --- New Response Parsing Logic --- + response_data = result['response_json'] + completion_data = {} + message = None + meta_query_id = None + + if response_data: + message = response_data.get('message') + data = response_data.get('data', {}) + meta_query_id = data.get('meta_query_id') + completions = data.get('completions', []) + if completions: + completion_data = completions[0] # Get the first completion + + log_entry = { + "timestamp": datetime.now().isoformat(), + "model_ids": json.dumps(payload['model_ids']), + "prefix": payload['context']['prefix'], + "suffix": payload['context']['suffix'], + "e2e_time_ms": result['e2e_time_ms'], + "error": result['error'], + "response_message": message, + "meta_query_id": meta_query_id, + "completion_model_id": completion_data.get('model_id'), + "completion_model_name": completion_data.get('model_name'), + "completion_text": completion_data.get('completion'), + "generation_time_ms": completion_data.get('generation_time'), + "confidence": completion_data.get('confidence') + } + all_results_data.append(log_entry) + + if (j + 1) % 25 == 0: + print(f" ...completed {j + 1}/{len(dataset)} requests.") + + if not all_results_data: + print("No data collected. Exiting.") + return + + df = pd.DataFrame(all_results_data) + + with sqlite3.connect(DB_NAME) as conn: + df.to_sql('results', conn, if_exists='append', index=False) + print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") + + df.to_csv(CSV_NAME, index=False) + print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") + + successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() + analyze_results(successful_timings) + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/performance_and_analysis/EvaluationScriptChat.py b/src/performance_and_analysis/EvaluationScriptChat.py index b75532c..09d3d2c 100644 --- a/src/performance_and_analysis/EvaluationScriptChat.py +++ b/src/performance_and_analysis/EvaluationScriptChat.py @@ -1,232 +1,232 @@ -import requests -import time -import os -import statistics -import math -from datasets import load_dataset -import sqlite3 -import json -from datetime import datetime -import pandas as pd -import uuid - -# --- Configuration --- -API_ENDPOINT = "http://localhost:8008/api/chat/request" -MODEL_IDS = [3] -PROMPT_PREFIX = "Please explain the different components of the solution for the following problem: " - -# --- Authentication Tokens --- -AUTH_TOKEN = os.environ.get('AUTH_TOKEN', '242dfc32-73b8-4a33-88dc-9279dd63b349') -PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', 'de68bfe7-55fc-4f7c-83cc-5a5901d3416e') -SESSION_TOKEN = os.environ.get('SESSION_TOKEN', '896e8387-bedf-4f2b-9f2c-6057ea37833d') - -# --- Benchmark Settings --- -DB_NAME = "chat_benchmark_results.db" -CSV_NAME = "chat_benchmark_results.csv" -NUM_WARMUP_RUNS = 5 -NUM_BENCHMARK_RUNS = 1 -BENCHMARK_SUBSET_SIZE = 100 # Use a subset for faster testing - - -# --- Helper Functions --- - -def setup_database(db_name="chat_benchmark_results.db"): - """Creates a SQLite database and a 'results' table for chat benchmarks.""" - conn = sqlite3.connect(db_name) - cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema - cursor.execute(''' - CREATE TABLE IF NOT EXISTS results ( - timestamp TEXT, - model_ids TEXT, - user_prompt TEXT, - e2e_time_ms REAL, - error TEXT, - chat_id TEXT, - response_title TEXT, - assistant_completion TEXT, - generation_time_ms INTEGER, - confidence REAL - ) - ''') - conn.commit() - conn.close() - print(f"Database '{db_name}' and new 'results' table are ready.") - - -def load_benchmark_data(subset_size=None): - """Loads the MBPP dataset from Hugging Face.""" - print("Downloading and preparing the MBPP dataset...") - try: - dataset = load_dataset("mbpp", "full")['test'] - if subset_size and subset_size < len(dataset): - print(f"Using a subset of {subset_size} samples.") - return dataset.select(range(subset_size)) - print(f"Using the full dataset with {len(dataset)} samples.") - return dataset - except Exception as e: - print(f"Error loading dataset: {e}") - return None - - -def prepare_request_payload(benchmark_item, model_ids): - """Formats a dataset item into the required chat JSON payload.""" - user_message = f"{PROMPT_PREFIX}{benchmark_item['text']}" - return { - "model_ids": model_ids, - "chat_id": str(uuid.uuid4()), - "messages": [ - ["system", "You are a helpful python assistant"], - ["user", user_message] - ], - # CORRECTED: Added all required fields with placeholder data - "context": { - "prefix": "", - "suffix": "", - "file_name": "benchmark.py", - "selected_text": "", - "context_files": [] - }, - "contextual_telemetry": { - "version_id": 1, - "trigger_type_id": 1, - "language_id": 1, - "file_path": "string", - "caret_line": 0, - "document_char_length": len(user_message), - "relative_document_position": 1 - }, - "behavioral_telemetry": { - "time_since_last_shown": 0, - "time_since_last_accepted": 0, - "typing_speed": 0 - }, - "web_enabled": False - } - -def send_request_and_measure_time(session, payload, cookies): - """Sends a request and returns a dictionary with time, response, and error.""" - start_time = time.perf_counter() - response_json, error_msg = None, None - try: - response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=60) # Increased timeout for chat - response.raise_for_status() - response_json = response.json() - except requests.exceptions.RequestException as e: - error_msg = str(e) - print(f"Request failed: {error_msg}") - finally: - end_time = time.perf_counter() - - duration_ms = (end_time - start_time) * 1000 - - return { - "e2e_time_ms": duration_ms, - "response_json": response_json, - "error": error_msg - } - - -def analyze_results(timings): - """Calculates and prints key statistics from the collected timings.""" - if not timings: - print("No successful requests to analyze.") - return - # This function remains the same as before - n = len(timings) - mean_val = statistics.mean(timings) - median_val = statistics.median(timings) - stdev_val = statistics.stdev(timings) if n > 1 else 0 - p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val - p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val - ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 - ci_lower = mean_val - ci_margin - ci_upper = mean_val + ci_margin - - print("\n--- Performance Analysis ---") - print(f"Total Successful Requests: {n}") - print(f"Mean Response Time: {mean_val:.2f} ms") - print(f"Median Response Time (P50):{median_val:.2f} ms") - print(f"95th Percentile (P95): {p95:.2f} ms") - print("--------------------------\n") - - -# --- Main Execution --- -def main(): - """Main function to run the benchmark.""" - print("Starting Chat End-to-End Response Time Benchmark...") - setup_database(DB_NAME) - - cookies = { - 'auth_token': AUTH_TOKEN, - 'project_token': PROJECT_TOKEN, - 'session_token': SESSION_TOKEN - } - - dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) - if not dataset: return - - with requests.Session() as session: - print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") - for i in range(NUM_WARMUP_RUNS): - payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) - send_request_and_measure_time(session, payload, cookies) - - all_results_data = [] - print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") - for i in range(NUM_BENCHMARK_RUNS): - print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") - for j, item in enumerate(dataset): - payload = prepare_request_payload(item, MODEL_IDS) - result = send_request_and_measure_time(session, payload, cookies) - - # --- New Response Parsing Logic for Chat --- - response_data = result['response_json'] - assistant_response = {} - chat_id, title = None, None - - if response_data: - chat_id = response_data.get('chat_id') - title = response_data.get('title') - history = response_data.get('history', []) - if history: - assistant_responses = history[-1].get('assistant_responses', []) - if assistant_responses: - assistant_response = assistant_responses[0] # Get the first assistant response - - log_entry = { - "timestamp": datetime.now().isoformat(), - "model_ids": json.dumps(payload['model_ids']), - "user_prompt": payload['messages'][-1][1], # Get the user message content - "e2e_time_ms": result['e2e_time_ms'], - "error": result['error'], - "chat_id": chat_id, - "response_title": title, - "assistant_completion": assistant_response.get('completion'), - "generation_time_ms": assistant_response.get('generation_time'), - "confidence": assistant_response.get('confidence') - } - all_results_data.append(log_entry) - - if (j + 1) % 10 == 0: - print(f" ...completed {j + 1}/{len(dataset)} requests.") - - if not all_results_data: - print("No data collected. Exiting.") - return - - df = pd.DataFrame(all_results_data) - - with sqlite3.connect(DB_NAME) as conn: - df.to_sql('results', conn, if_exists='append', index=False) - print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") - - df.to_csv(CSV_NAME, index=False) - print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") - - successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() - analyze_results(successful_timings) - - -if __name__ == "__main__": +import requests +import time +import os +import statistics +import math +from datasets import load_dataset +import sqlite3 +import json +from datetime import datetime +import pandas as pd +import uuid + +# --- Configuration --- +API_ENDPOINT = "http://localhost:8008/api/chat/request" +MODEL_IDS = [3] +PROMPT_PREFIX = "Please explain the different components of the solution for the following problem: " + +# --- Authentication Tokens --- +AUTH_TOKEN = os.environ.get('AUTH_TOKEN', '242dfc32-73b8-4a33-88dc-9279dd63b349') +PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', 'de68bfe7-55fc-4f7c-83cc-5a5901d3416e') +SESSION_TOKEN = os.environ.get('SESSION_TOKEN', '896e8387-bedf-4f2b-9f2c-6057ea37833d') + +# --- Benchmark Settings --- +DB_NAME = "chat_benchmark_results.db" +CSV_NAME = "chat_benchmark_results.csv" +NUM_WARMUP_RUNS = 5 +NUM_BENCHMARK_RUNS = 1 +BENCHMARK_SUBSET_SIZE = 100 # Use a subset for faster testing + + +# --- Helper Functions --- + +def setup_database(db_name="chat_benchmark_results.db"): + """Creates a SQLite database and a 'results' table for chat benchmarks.""" + conn = sqlite3.connect(db_name) + cursor = conn.cursor() + cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema + cursor.execute(''' + CREATE TABLE IF NOT EXISTS results ( + timestamp TEXT, + model_ids TEXT, + user_prompt TEXT, + e2e_time_ms REAL, + error TEXT, + chat_id TEXT, + response_title TEXT, + assistant_completion TEXT, + generation_time_ms INTEGER, + confidence REAL + ) + ''') + conn.commit() + conn.close() + print(f"Database '{db_name}' and new 'results' table are ready.") + + +def load_benchmark_data(subset_size=None): + """Loads the MBPP dataset from Hugging Face.""" + print("Downloading and preparing the MBPP dataset...") + try: + dataset = load_dataset("mbpp", "full")['test'] + if subset_size and subset_size < len(dataset): + print(f"Using a subset of {subset_size} samples.") + return dataset.select(range(subset_size)) + print(f"Using the full dataset with {len(dataset)} samples.") + return dataset + except Exception as e: + print(f"Error loading dataset: {e}") + return None + + +def prepare_request_payload(benchmark_item, model_ids): + """Formats a dataset item into the required chat JSON payload.""" + user_message = f"{PROMPT_PREFIX}{benchmark_item['text']}" + return { + "model_ids": model_ids, + "chat_id": str(uuid.uuid4()), + "messages": [ + ["system", "You are a helpful python assistant"], + ["user", user_message] + ], + # CORRECTED: Added all required fields with placeholder data + "context": { + "prefix": "", + "suffix": "", + "file_name": "benchmark.py", + "selected_text": "", + "context_files": [] + }, + "contextual_telemetry": { + "version_id": 1, + "trigger_type_id": 1, + "language_id": 1, + "file_path": "string", + "caret_line": 0, + "document_char_length": len(user_message), + "relative_document_position": 1 + }, + "behavioral_telemetry": { + "time_since_last_shown": 0, + "time_since_last_accepted": 0, + "typing_speed": 0 + }, + "web_enabled": False + } + +def send_request_and_measure_time(session, payload, cookies): + """Sends a request and returns a dictionary with time, response, and error.""" + start_time = time.perf_counter() + response_json, error_msg = None, None + try: + response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=60) # Increased timeout for chat + response.raise_for_status() + response_json = response.json() + except requests.exceptions.RequestException as e: + error_msg = str(e) + print(f"Request failed: {error_msg}") + finally: + end_time = time.perf_counter() + + duration_ms = (end_time - start_time) * 1000 + + return { + "e2e_time_ms": duration_ms, + "response_json": response_json, + "error": error_msg + } + + +def analyze_results(timings): + """Calculates and prints key statistics from the collected timings.""" + if not timings: + print("No successful requests to analyze.") + return + # This function remains the same as before + n = len(timings) + mean_val = statistics.mean(timings) + median_val = statistics.median(timings) + stdev_val = statistics.stdev(timings) if n > 1 else 0 + p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val + p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val + ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 + ci_lower = mean_val - ci_margin + ci_upper = mean_val + ci_margin + + print("\n--- Performance Analysis ---") + print(f"Total Successful Requests: {n}") + print(f"Mean Response Time: {mean_val:.2f} ms") + print(f"Median Response Time (P50):{median_val:.2f} ms") + print(f"95th Percentile (P95): {p95:.2f} ms") + print("--------------------------\n") + + +# --- Main Execution --- +def main(): + """Main function to run the benchmark.""" + print("Starting Chat End-to-End Response Time Benchmark...") + setup_database(DB_NAME) + + cookies = { + 'auth_token': AUTH_TOKEN, + 'project_token': PROJECT_TOKEN, + 'session_token': SESSION_TOKEN + } + + dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) + if not dataset: return + + with requests.Session() as session: + print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") + for i in range(NUM_WARMUP_RUNS): + payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) + send_request_and_measure_time(session, payload, cookies) + + all_results_data = [] + print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") + for i in range(NUM_BENCHMARK_RUNS): + print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") + for j, item in enumerate(dataset): + payload = prepare_request_payload(item, MODEL_IDS) + result = send_request_and_measure_time(session, payload, cookies) + + # --- New Response Parsing Logic for Chat --- + response_data = result['response_json'] + assistant_response = {} + chat_id, title = None, None + + if response_data: + chat_id = response_data.get('chat_id') + title = response_data.get('title') + history = response_data.get('history', []) + if history: + assistant_responses = history[-1].get('assistant_responses', []) + if assistant_responses: + assistant_response = assistant_responses[0] # Get the first assistant response + + log_entry = { + "timestamp": datetime.now().isoformat(), + "model_ids": json.dumps(payload['model_ids']), + "user_prompt": payload['messages'][-1][1], # Get the user message content + "e2e_time_ms": result['e2e_time_ms'], + "error": result['error'], + "chat_id": chat_id, + "response_title": title, + "assistant_completion": assistant_response.get('completion'), + "generation_time_ms": assistant_response.get('generation_time'), + "confidence": assistant_response.get('confidence') + } + all_results_data.append(log_entry) + + if (j + 1) % 10 == 0: + print(f" ...completed {j + 1}/{len(dataset)} requests.") + + if not all_results_data: + print("No data collected. Exiting.") + return + + df = pd.DataFrame(all_results_data) + + with sqlite3.connect(DB_NAME) as conn: + df.to_sql('results', conn, if_exists='append', index=False) + print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") + + df.to_csv(CSV_NAME, index=False) + print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") + + successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() + analyze_results(successful_timings) + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/performance_and_analysis/benchmark_results.csv b/src/performance_and_analysis/benchmark_results.csv index 2ac3ee7..81a56b7 100644 --- a/src/performance_and_analysis/benchmark_results.csv +++ b/src/performance_and_analysis/benchmark_results.csv @@ -1,4 +1,4 @@ -timestamp,model_ids,prefix,suffix,e2e_time_ms,error,response_message,meta_query_id,completion_model_id,completion_model_name,completion_text,generation_time_ms,confidence +timestamp,model_ids,prefix,suffix,e2e_time_ms,error,response_message,meta_query_id,completion_model_id,completion_model_name,completion_text,generation_time_ms,confidence 2025-09-22T13:56:39.060324,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27,7 +27,7 @@ class Problem { } }",84.04280000104336,,Completions generated successfully. Ready for display in your IDE.,6c9846fd-58a5-48f2-b0e3-f93c40e555c6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:39.226354,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -61,7 +61,7 @@ class Problem { }"," } }",165.78080000181217,,Completions generated successfully. Ready for display in your IDE.,a3d0511a-e555-4563-acf1-1c062607adfc,1,deepseek-ai/deepseek-coder-1.3b-base," return new String(arr); -",116,1.0 +",116,1.0 2025-09-22T13:56:39.314124,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -89,7 +89,7 @@ class Problem { } }",87.12779999768827,,Completions generated successfully. Ready for display in your IDE.,50c4e0e5-1f5f-4446-a7b1-154c362686db,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:39.399312,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -114,7 +114,7 @@ class Problem { return True else:"," ",85.44360000087181,,Completions generated successfully. Ready for display in your IDE.,cf823418-b1c8-46f1-9c06-1bcf29fe6856,1,deepseek-ai/deepseek-coder-1.3b-base," - return False",61,1.0 + return False",61,1.0 2025-09-22T13:56:39.535994,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -154,7 +154,7 @@ class Problem { } }",136.19849999668077,,Completions generated successfully. Ready for display in your IDE.,d9229a9e-9def-4715-bac6-24019eab32ef,1,deepseek-ai/deepseek-coder-1.3b-base," int sum = 0; -",111,1.0 +",111,1.0 2025-09-22T13:56:39.636250,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -191,7 +191,7 @@ function move_one_ball(arr){ else{ return true;",},100.04470000421861,,Completions generated successfully. Ready for display in your IDE.,4c9f998b-4f1f-4d64-a5b8-d98df8b59650,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:39.855396,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -258,7 +258,7 @@ function numerical_letter_grade(grades){ return grades_array; }",219.43630000168923,,Completions generated successfully. Ready for display in your IDE.,953e6f2c-904c-4c32-b069-bd7557a482ef,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 1.7){ -",191,1.0 +",191,1.0 2025-09-22T13:56:39.981516,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -306,7 +306,7 @@ class Problem { } }",125.68989999999758,,Completions generated successfully. Ready for display in your IDE.,8aca7d62-3fac-4050-875c-d72bead419b8,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",97,1.0 +",97,1.0 2025-09-22T13:56:40.328580,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -340,7 +340,7 @@ class Problem { }",346.68429999874206,,Completions generated successfully. Ready for display in your IDE.,b6a4008b-70fa-458b-bcdb-0800281c3db4,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < words.length; ++i) { String word = words[i]; -",320,1.0 +",320,1.0 2025-09-22T13:56:40.401109,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -379,7 +379,7 @@ class Problem { } return res; }",,72.13529999717139,,Completions generated successfully. Ready for display in your IDE.,33e1d8df-e2f8-4d93-8972-01cfec8444d9,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:56:40.595166,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -421,7 +421,7 @@ function compare_one(a, b){ } ",194.15789999766275,,Completions generated successfully. Ready for display in your IDE.,a27f73c3-6dce-41db-96c0-5bbb891d8d88,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (a < b[0]){ -",169,1.0 +",169,1.0 2025-09-22T13:56:40.663832,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -464,7 +464,7 @@ class Problem { return list.stream().mapToLong(x -> x).sum(); } }",68.723600001249,,Completions generated successfully. Ready for display in your IDE.,1e998f85-39b0-475b-8047-5feb652b761a,1,deepseek-ai/deepseek-coder-1.3b-base," -",40,1.0 +",40,1.0 2025-09-22T13:56:40.912634,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -529,7 +529,7 @@ class Problem { } }",248.43999999575317,,Completions generated successfully. Ready for display in your IDE.,6d79b105-60f9-4822-8051-a532ff8c4765,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 3.7f) { -",218,1.0 +",218,1.0 2025-09-22T13:56:41.194537,[1],"def fibfib(n: int) -> int: """"""The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fibfib(0) == 0 @@ -552,7 +552,7 @@ class Problem { a = b b = c c = a+b+c -",228,0.9409 +",228,0.9409 2025-09-22T13:56:41.341645,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -588,7 +588,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",146.48559999477584,,Completions generated successfully. Ready for display in your IDE.,5109a05c-1e5e-449d-9b08-ab2b84b0b408,1,deepseek-ai/deepseek-coder-1.3b-base," elif c == ')': -",121,1.0 +",121,1.0 2025-09-22T13:56:41.439420,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -611,7 +611,7 @@ class Problem { public static long doubleTheDifference(ArrayList lst) { return lst.stream().filter(x -> x%2 == 1).mapToLong(Math::round).map(x -> x*x).sum();",},97.91650000261143,,Completions generated successfully. Ready for display in your IDE.,b525cc66-8b21-4d01-a80b-83d7957df4ad,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:56:41.528780,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -631,7 +631,7 @@ class Problem { } return l;",},89.21879999979865,,Completions generated successfully. Ready for display in your IDE.,6ef6f81d-e805-4144-83b9-5389ff0f6fd1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:56:41.746076,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -648,7 +648,7 @@ function largest_prime_factor(n){ return result; }",216.6819000049145,,Completions generated successfully. Ready for display in your IDE.,4b1c7f9d-8b89-4082-ab7f-b3d4c9558e9e,1,deepseek-ai/deepseek-coder-1.3b-base," if (n % result == 0 && checkPrime(result)){ -",194,0.9823 +",194,0.9823 2025-09-22T13:56:42.000695,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -678,7 +678,7 @@ class Problem { } }",254.75179999921238,,Completions generated successfully. Ready for display in your IDE.,069191f0-e801-4955-8c12-4e0a9c459fe9,1,deepseek-ai/deepseek-coder-1.3b-base," for (long i = 0; i < n; i++) { -",206,1.0 +",206,1.0 2025-09-22T13:56:42.124004,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -714,7 +714,7 @@ class Problem { ",122.95789999916451,,Completions generated successfully. Ready for display in your IDE.,2c465269-391b-4268-a22c-b7a752f2ef28,1,deepseek-ai/deepseek-coder-1.3b-base," number %= key -",98,1.0 +",98,1.0 2025-09-22T13:56:42.218937,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -754,7 +754,7 @@ class Problem { } }",94.39610000117682,,Completions generated successfully. Ready for display in your IDE.,1f70298d-82a1-474e-83fc-effcc41c9282,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:56:42.348577,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -789,7 +789,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: else: sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",129.5250999974087,,Completions generated successfully. Ready for display in your IDE.,1849287b-b1a2-4ee3-99f7-2911aab1c19c,1,deepseek-ai/deepseek-coder-1.3b-base," def product(lst): -",103,0.9896 +",103,0.9896 2025-09-22T13:56:42.444638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -839,7 +839,7 @@ class Problem { } }",96.230100003595,,Completions generated successfully. Ready for display in your IDE.,b9b41a71-2c2b-4f8f-9b45-ef9805db9709,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:42.685087,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -863,7 +863,7 @@ function vowels_count(s){ } ",240.5473000035272,,Completions generated successfully. Ready for display in your IDE.,2b192ae7-4a1c-422f-94e9-7015ba78da2f,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0; i< s.length ; ++i) { -",217,0.9208 +",217,0.9208 2025-09-22T13:56:42.865405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -897,7 +897,7 @@ class Problem { } }",179.4445000050473,,Completions generated successfully. Ready for display in your IDE.,dc02a6be-00b2-45ae-ace8-3f62e5417624,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (c == '>') { -",155,0.9933 +",155,0.9933 2025-09-22T13:56:42.960302,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -937,7 +937,7 @@ class Problem { } return arrayList;",},94.71489999850746,,Completions generated successfully. Ready for display in your IDE.,9edc92be-944b-4923-946b-e53c35d5b293,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:43.232458,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -979,7 +979,7 @@ class Problem { } }",271.9797999961884,,Completions generated successfully. Ready for display in your IDE.,8bf973ef-40bc-4aaa-ab4f-218c71f68ff5,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < s3.length(); i++) { -",244,1.0 +",244,1.0 2025-09-22T13:56:43.389957,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1019,7 +1019,7 @@ class Problem { } }",156.99749999475898,,Completions generated successfully. Ready for display in your IDE.,60b1b9f9-3f8a-4d41-909f-f7e49145179d,1,deepseek-ai/deepseek-coder-1.3b-base," boolean isPrime = true; -",131,1.0 +",131,1.0 2025-09-22T13:56:43.749958,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -1060,7 +1060,7 @@ function max_fill(grid, capacity){ } else{ break; } -",335,1.0 +",335,1.0 2025-09-22T13:56:43.837792,[1],"//Write a function that takes a string and returns true if the string // length is a prime number or false otherwise // Examples @@ -1080,7 +1080,7 @@ function prime_length(string){ } else { return false;",},88.09750000364147,,Completions generated successfully. Ready for display in your IDE.,4f9eff4e-f371-45a7-b9e7-d388150e6ec5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:56:43.990629,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -1105,7 +1105,7 @@ function sort_numbers(numbers){ return numString; }",152.56110000336776,,Completions generated successfully. Ready for display in your IDE.,dc399fd6-ec2d-4ab8-a818-9633740c4ae6,1,deepseek-ai/deepseek-coder-1.3b-base," ""one"": 1, -",114,1.0 +",114,1.0 2025-09-22T13:56:44.092477,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1154,7 +1154,7 @@ class Problem { } return s;",},100.99959999934072,,Completions generated successfully. Ready for display in your IDE.,498eaf2b-680f-4a69-9862-9ae59137550f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:44.189428,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -1176,7 +1176,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: elif paren == ')': current_depth -= 1 result.append(max_depth)",,96.98519999801647,,Completions generated successfully. Ready for display in your IDE.,393380e5-1edb-41b1-a0e8-f79f18d611cf,1,deepseek-ai/deepseek-coder-1.3b-base," - return result",72,0.9232 + return result",72,0.9232 2025-09-22T13:56:44.449626,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -1218,7 +1218,7 @@ function int_to_mini_roman(number){ ",260.3151000002981,,Completions generated successfully. Ready for display in your IDE.,f459d96e-0827-427c-bb2d-4a5b4567b34f,1,deepseek-ai/deepseek-coder-1.3b-base," //convert the given input to an array of keys from largest to smallest value -",234,0.8907 +",234,0.8907 2025-09-22T13:56:44.546273,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -1273,7 +1273,7 @@ function by_length(arr){ return arr3; }",96.39830000378424,,Completions generated successfully. Ready for display in your IDE.,149eefe0-5fa7-4fac-84aa-5b04ac28a30d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:56:44.741128,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -1341,7 +1341,7 @@ function do_algebra(operator, operand){ } ",194.2263000019011,,Completions generated successfully. Ready for display in your IDE.,e81dc2b1-a2bc-4143-99bb-49acaeb6c79f,1,deepseek-ai/deepseek-coder-1.3b-base," operator.splice(count, 1); -",165,1.0 +",165,1.0 2025-09-22T13:56:44.840098,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -1375,7 +1375,7 @@ function count_nums(arr){ } }",},99.40949999872828,,Completions generated successfully. Ready for display in your IDE.,d89e1267-cd4c-4a04-ada4-6675159f79da,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",73,1.0 +",73,1.0 2025-09-22T13:56:45.021452,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -1442,7 +1442,7 @@ function numerical_letter_grade(grades){ return grades_array; }",180.2936999956728,,Completions generated successfully. Ready for display in your IDE.,bfd77b3e-f522-43c9-ae2b-83ba501b351c,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""E""); -",149,1.0 +",149,1.0 2025-09-22T13:56:45.303716,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -1483,7 +1483,7 @@ function valid_date(date){ if(!isLeapYear(year)){ return false; } else { -",255,0.9693 +",255,0.9693 2025-09-22T13:56:45.542590,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -1555,7 +1555,7 @@ function minPath(grid, k){ return path; }",239.0395999973407,,Completions generated successfully. Ready for display in your IDE.,c118f5b9-de89-4146-a18f-2416db951be9,1,deepseek-ai/deepseek-coder-1.3b-base," neighbours.push(grid[x + 1][y]); -",208,1.0 +",208,1.0 2025-09-22T13:56:45.604776,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -1591,7 +1591,7 @@ function max_fill(grid, capacity){ return maxFill; } ",61.79560000600759,,Completions generated successfully. Ready for display in your IDE.,b41f4acd-b5ac-4c50-9ee8-38b4a929be82,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T13:56:45.910099,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1639,7 +1639,7 @@ class Problem { } }",304.7714999993332,,Completions generated successfully. Ready for display in your IDE.,1e4f30ca-cef3-4d5c-a894-cd3fd26809fa,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.abs(indexPlanet1 - indexPlanet2) < 8) { -",274,0.9484 +",274,0.9484 2025-09-22T13:56:46.124406,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1688,7 +1688,7 @@ class Problem { } }",213.55380000022706,,Completions generated successfully. Ready for display in your IDE.,5787dbb1-9662-4f60-b0f2-6a9e152f0500,1,deepseek-ai/deepseek-coder-1.3b-base," if (max < hm.get(key)) { -",186,1.0 +",186,1.0 2025-09-22T13:56:46.250977,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -1710,7 +1710,7 @@ function modp(n, p){ ",126.74230000266107,,Completions generated successfully. Ready for display in your IDE.,809024ed-0f7b-4369-8f5f-59bfa147e192,1,deepseek-ai/deepseek-coder-1.3b-base," while n > 0 { -",104,0.8889 +",104,0.8889 2025-09-22T13:56:46.352337,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1771,7 +1771,7 @@ class Problem { } } }",101.11350000079256,,Completions generated successfully. Ready for display in your IDE.,24370b93-3bbf-4c6c-b5a6-f56641823959,1,deepseek-ai/deepseek-coder-1.3b-base," - ",70,1.0 + ",70,1.0 2025-09-22T13:56:46.427267,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -1810,7 +1810,7 @@ function find_max(words){ } } return max_unique_word;",,74.26700000360142,,Completions generated successfully. Ready for display in your IDE.,c6e85889-ef41-467a-940d-18d7cde96bda,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:56:46.470697,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1843,7 +1843,7 @@ class Problem { } return fib; } -}",43.263400002615526,,Completions generated successfully. Ready for display in your IDE.,2213393e-3bd4-458f-988d-733728beb17d,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",43.263400002615526,,Completions generated successfully. Ready for display in your IDE.,2213393e-3bd4-458f-988d-733728beb17d,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:56:46.658609,[1],"from typing import List def get_odd_collatz(n: int) -> List[int]: @@ -1877,7 +1877,7 @@ def get_odd_collatz(n: int) -> List[int]: ",187.7396999989287,,Completions generated successfully. Ready for display in your IDE.,f920c153-78e0-4e3f-8c54-dcfc7414c5c7,1,deepseek-ai/deepseek-coder-1.3b-base," current_num = current_num // 2 -",163,0.9487 +",163,0.9487 2025-09-22T13:56:46.723717,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1902,7 +1902,7 @@ class Problem { public static boolean sameChars(String s0, String s1) { return s0.chars().distinct().sorted().boxed().collect(Collectors.toList()).equals(s1.chars().distinct().sorted().boxed().collect(Collectors.toList())); }",,65.14099999913014,,Completions generated successfully. Ready for display in your IDE.,f138df19-15c6-4d0d-ae1f-6c393d104fed,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:56:46.862902,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -1926,7 +1926,7 @@ function vowels_count(s){ } ",138.54729999729898,,Completions generated successfully. Ready for display in your IDE.,b8a1037e-6c23-4b0a-b984-23a6a355b5d0,1,deepseek-ai/deepseek-coder-1.3b-base," count+=1; -",115,0.9924 +",115,0.9924 2025-09-22T13:56:46.969579,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1950,7 +1950,7 @@ class Problem { } }",106.65840000001481,,Completions generated successfully. Ready for display in your IDE.,45f6acbd-cdea-4dda-8935-38380d9203dd,1,deepseek-ai/deepseek-coder-1.3b-base," return x; -",83,1.0 +",83,1.0 2025-09-22T13:56:47.667384,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -1980,7 +1980,7 @@ for (let i= 2;i= 0; i-- ){ -",269,0.9012 +",269,0.9012 2025-09-22T13:56:48.504102,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -2097,7 +2097,7 @@ function separate_paren_groups(paren_string){ group = ''; } } - return groups;",,44.200899996212684,,Completions generated successfully. Ready for display in your IDE.,99ed0b90-bd29-4100-a4c9-83f6f30374cc,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.9317 + return groups;",,44.200899996212684,,Completions generated successfully. Ready for display in your IDE.,99ed0b90-bd29-4100-a4c9-83f6f30374cc,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.9317 2025-09-22T13:56:48.602308,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2147,7 +2147,7 @@ class Problem { } return ret;",},98.49430000031134,,Completions generated successfully. Ready for display in your IDE.,05cf2979-9bfe-4738-83e4-ccefe397eccd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:48.740247,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -2174,7 +2174,7 @@ function file_name_check(file_name){ } ",137.46569999784697,,Completions generated successfully. Ready for display in your IDE.,15b14cf6-9c11-40f6-aa35-65425e49ebc1,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",111,0.8412 +",111,0.8412 2025-09-22T13:56:48.817918,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2204,7 +2204,7 @@ class Problem { } return res; }",,76.87330000044312,,Completions generated successfully. Ready for display in your IDE.,b1086b67-ef16-4dc4-bb46-98f3ca0edffc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:56:48.953648,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -2238,7 +2238,7 @@ class Problem { prime_words.append(word) return ' '.join(prime_words)",135.96839999809163,,Completions generated successfully. Ready for display in your IDE.,f91eb261-a19c-4cda-92c6-088397fbe62d,1,deepseek-ai/deepseek-coder-1.3b-base," for word in words: -",111,1.0 +",111,1.0 2025-09-22T13:56:49.179358,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -2275,7 +2275,7 @@ function max_fill(grid, capacity){ } ",225.1678999964497,,Completions generated successfully. Ready for display in your IDE.,39d8c21e-aa2b-42b1-ab61-3bf91ae1208b,1,deepseek-ai/deepseek-coder-1.3b-base," for(var j = 0; j < c; j++){ -",202,0.9875 +",202,0.9875 2025-09-22T13:56:49.297163,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2319,7 +2319,7 @@ class Problem { }"," } }",118.0813000028138,,Completions generated successfully. Ready for display in your IDE.,d06cb787-f4ae-4da4-9609-ddabd2eaf635,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",91,1.0 +",91,1.0 2025-09-22T13:56:49.364040,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -2338,7 +2338,7 @@ function unique_digits(x){ } return result.sort(function(a, b){return a-b})"," ",66.50459999946179,,Completions generated successfully. Ready for display in your IDE.,4b60bb17-a5ac-429f-bf4e-19181cdc8d3d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:56:49.469131,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -2364,7 +2364,7 @@ function correct_bracketing(brackets){ } ",104.71849999885308,,Completions generated successfully. Ready for display in your IDE.,26390f0f-4b16-49e7-afcb-d25d7a5229d1,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:56:49.760619,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2399,7 +2399,7 @@ class Problem { } }",291.0307999991346,,Completions generated successfully. Ready for display in your IDE.,b8311c84-f730-4cfa-b220-d7f1fff53558,1,deepseek-ai/deepseek-coder-1.3b-base," if (n % i == 0 || n % (i + 2) == 0) return false; -",267,1.0 +",267,1.0 2025-09-22T13:56:49.952525,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2428,7 +2428,7 @@ class Problem { } }",191.91030000365572,,Completions generated successfully. Ready for display in your IDE.,270a67a5-914a-454c-bc19-961bccb4f543,1,deepseek-ai/deepseek-coder-1.3b-base," if ((n & 1) == 0) { -",165,0.9784 +",165,0.9784 2025-09-22T13:56:50.233063,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -2500,7 +2500,7 @@ function minPath(grid, k){ return path; }",280.65820000483654,,Completions generated successfully. Ready for display in your IDE.,408df907-0480-4b71-b794-f55468ed9480,1,deepseek-ai/deepseek-coder-1.3b-base," for(let i = 1; i < neighbours.length; i++){ -",247,1.0 +",247,1.0 2025-09-22T13:56:50.362668,[1],"from typing import List, Tuple def sum_product(numbers: List[int]) -> Tuple[int, int]: @@ -2517,7 +2517,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: return (sum, product) ",128.79180000163615,,Completions generated successfully. Ready for display in your IDE.,e6b00e05-e1d0-42c2-b13f-534ebe14e0c3,1,deepseek-ai/deepseek-coder-1.3b-base," for i in numbers: -",105,1.0 +",105,1.0 2025-09-22T13:56:50.451609,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2548,7 +2548,7 @@ class Problem { } }",88.98670000053244,,Completions generated successfully. Ready for display in your IDE.,cdfe94d8-3bd7-48cf-9b58-c70bda248308,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:56:50.547331,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -2586,7 +2586,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",95.63859999616398,,Completions generated successfully. Ready for display in your IDE.,356ce3f7-32ff-45b5-8eb7-688ab087bf01,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:50.706707,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -2606,7 +2606,7 @@ function unique_digits(x){ ",159.0252000023611,,Completions generated successfully. Ready for display in your IDE.,3c1ca8eb-3327-4606-8ecc-92ced91856d3,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""result"", result) -",135,0.9736 +",135,0.9736 2025-09-22T13:56:50.824436,[1],"from typing import List, Tuple def sum_product(numbers: List[int]) -> Tuple[int, int]: @@ -2623,7 +2623,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: sum += i"," return (sum, product) ",117.29550000018207,,Completions generated successfully. Ready for display in your IDE.,634d9f3e-dfea-4c0d-a07e-94742a9fcc2d,1,deepseek-ai/deepseek-coder-1.3b-base," product *= i -",92,1.0 +",92,1.0 2025-09-22T13:56:50.982047,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -2649,7 +2649,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: ",158.13349999370985,,Completions generated successfully. Ready for display in your IDE.,24f3332c-4266-4164-93bf-a7aa9ba39361,1,deepseek-ai/deepseek-coder-1.3b-base," if n % d == 0: -",127,0.9916 +",127,0.9916 2025-09-22T13:56:51.111919,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -2675,7 +2675,7 @@ def unique_digits(x: List[int]) -> List[int]: return sorted(x) ",128.98670000140555,,Completions generated successfully. Ready for display in your IDE.,20016a6c-14cc-49ee-9bb9-3d0eff4c3bb5,1,deepseek-ai/deepseek-coder-1.3b-base," for i in x: -",104,1.0 +",104,1.0 2025-09-22T13:56:51.288131,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -2701,7 +2701,7 @@ function is_multiply_prime(a){ } }",176.51130000012927,,Completions generated successfully. Ready for display in your IDE.,d0777c38-9d52-4b07-bc38-84b0c6282dba,1,deepseek-ai/deepseek-coder-1.3b-base," check.push(prime[i]); -",135,1.0 +",135,1.0 2025-09-22T13:56:51.383547,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -2751,7 +2751,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",94.84629999496974,,Completions generated successfully. Ready for display in your IDE.,09b7370a-6f88-47fe-b960-5c76abb6abef,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:51.695781,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -2823,7 +2823,7 @@ function minPath(grid, k){ return path; }",311.713700000837,,Completions generated successfully. Ready for display in your IDE.,12e908f2-035b-4877-9478-1f8de1b6a904,1,deepseek-ai/deepseek-coder-1.3b-base," for(let i = 0; i < n; i++){ -",219,1.0 +",219,1.0 2025-09-22T13:56:51.999054,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2843,7 +2843,7 @@ class Problem { "," return (float)numbers.stream().mapToDouble(n -> Math.abs(n - mean)).average().getAsDouble(); } }",303.8187000056496,,Completions generated successfully. Ready for display in your IDE.,47f1d2f7-0928-4382-bb5f-6ec99a022610,1,deepseek-ai/deepseek-coder-1.3b-base," double mean = numbers.stream().mapToDouble(n -> n).average().getAsDouble(); -",279,0.9914 +",279,0.9914 2025-09-22T13:56:52.416763,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -2866,7 +2866,7 @@ function car_race_collision(n){ ",416.74769999372074,,Completions generated successfully. Ready for display in your IDE.,7afc3288-278b-4fda-949a-2b28e33d7e77,1,deepseek-ai/deepseek-coder-1.3b-base," // If both cars are moving towards eachother... if ((i != j && Math.abs((j - i)) % 2 == 0)){ -",393,0.9384 +",393,0.9384 2025-09-22T13:56:52.573224,[1],"def iscube(a: int) -> bool: """""" Write a function that takes an integer a and returns True @@ -2896,7 +2896,7 @@ function car_race_collision(n){ ",156.81169999879785,,Completions generated successfully. Ready for display in your IDE.,ee272af2-2b7c-4b94-8be3-a7d54e54aa06,1,deepseek-ai/deepseek-coder-1.3b-base," if (i**3 == a): -",134,0.9205 +",134,0.9205 2025-09-22T13:56:52.685466,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -2923,7 +2923,7 @@ function strange_sort_list(lst){ tmp.splice(tmp.indexOf(output[i]), 1); }",},111.35720000311267,,Completions generated successfully. Ready for display in your IDE.,7df4749f-07f4-4b41-b51f-5f309cb7bcc1,1,deepseek-ai/deepseek-coder-1.3b-base," return output; -",83,1.0 +",83,1.0 2025-09-22T13:56:52.811134,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2973,7 +2973,7 @@ class Problem { } }",125.77549999696203,,Completions generated successfully. Ready for display in your IDE.,9ee87c77-d441-4ed0-a9aa-7756723ed6bf,1,deepseek-ai/deepseek-coder-1.3b-base," case 'o': -",99,0.9913 +",99,0.9913 2025-09-22T13:56:52.995085,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3013,7 +3013,7 @@ class Problem { } }",183.61670000012964,,Completions generated successfully. Ready for display in your IDE.,ebe82e13-6407-4063-b550-ca96e4f87dce,1,deepseek-ai/deepseek-coder-1.3b-base," b.add(x.get(i)); -",159,1.0 +",159,1.0 2025-09-22T13:56:53.059168,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -3044,7 +3044,7 @@ function hex_key(num){ } return result; }",64.06440000137081,,Completions generated successfully. Ready for display in your IDE.,11abd296-dc7c-4731-86c0-e675dfcb2eb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T13:56:53.131429,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3089,7 +3089,7 @@ class Problem { } return ans.toString(); }",,71.7365999953472,,Completions generated successfully. Ready for display in your IDE.,3e4ef572-4fb3-4ade-ab88-9d16a4928035,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:56:53.424821,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3137,7 +3137,7 @@ class Problem { } }",292.9761000050348,,Completions generated successfully. Ready for display in your IDE.,2021873e-998c-41f0-be12-edbc29ec71ab,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < l_r.size(); i++) { -",264,0.9912 +",264,0.9912 2025-09-22T13:56:53.519369,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -3156,7 +3156,7 @@ function count_upper(s){ }"," return result; }",94.36060000007274,,Completions generated successfully. Ready for display in your IDE.,cf0b46d5-61c1-480e-b1bb-5cb483666b57,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:54.215691,[1],"//Concatenate array of strings into a single string // >>> concatenate([]) // """" @@ -3172,7 +3172,7 @@ function concatenate(strings){ for(var i = 0 ;i < strings.length - 2; ++i ) { newString += strings[i]; }; - //console.log(""new String",672,0.9483 + //console.log(""new String",672,0.9483 2025-09-22T13:56:54.357060,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -3196,7 +3196,7 @@ function parse_music(music_string){ } ",140.82329999655485,,Completions generated successfully. Ready for display in your IDE.,e35d1ef2-13f4-47bd-8763-fd49b8d2aede,1,deepseek-ai/deepseek-coder-1.3b-base," song.push(2); -",114,1.0 +",114,1.0 2025-09-22T13:56:54.452736,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -3233,7 +3233,7 @@ function parse_music(music_string){ return False if len(str(date_list[2])) != 4: return False",,95.41989999706857,,Completions generated successfully. Ready for display in your IDE.,dbea6019-4710-4b7c-88f7-868d0a0cee9a,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",69,1.0 + return True",69,1.0 2025-09-22T13:56:54.559829,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -3270,7 +3270,7 @@ function parse_music(music_string){ return False return True",106.43789999448927,,Completions generated successfully. Ready for display in your IDE.,0b150836-c064-4ad5-a001-a96781490a03,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",80,1.0 +",80,1.0 2025-09-22T13:56:54.686275,[1],"//The Brazilian factorial is defined as: // brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! // where n > 0 @@ -3290,7 +3290,7 @@ function special_factorial(n){ } ",126.38159999914933,,Completions generated successfully. Ready for display in your IDE.,04a6080d-6f2e-4569-b8a3-8ef2ba373eed,1,deepseek-ai/deepseek-coder-1.3b-base," let fact = 1; -",102,1.0 +",102,1.0 2025-09-22T13:56:54.854222,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -3318,7 +3318,7 @@ function fib4(n){ } ",168.04359999514418,,Completions generated successfully. Ready for display in your IDE.,67bf8d5f-c0a6-4b0e-a42e-4e8b736a1d00,1,deepseek-ai/deepseek-coder-1.3b-base," prev[2] = prev[3]; -",144,1.0 +",144,1.0 2025-09-22T13:56:54.936791,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -3332,7 +3332,7 @@ function largest_divisor(n){ return ld; }",82.43550000042887,,Completions generated successfully. Ready for display in your IDE.,619b3eda-f634-4296-9d27-8f2610da5b0b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:54.993054,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -3348,7 +3348,7 @@ function has_close_elements(numbers, threshold){ } return false; } -",55.56539999815868,,Completions generated successfully. Ready for display in your IDE.,e66237e2-d116-462c-9c95-fab76c67b999,1,deepseek-ai/deepseek-coder-1.3b-base, ,35,0.3293 +",55.56539999815868,,Completions generated successfully. Ready for display in your IDE.,e66237e2-d116-462c-9c95-fab76c67b999,1,deepseek-ai/deepseek-coder-1.3b-base, ,35,0.3293 2025-09-22T13:56:55.445198,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3392,7 +3392,7 @@ class Problem { } }",452.2540999969351,,Completions generated successfully. Ready for display in your IDE.,1fce02a4-6afb-4303-9cb5-bd9c81e7bbe9,1,deepseek-ai/deepseek-coder-1.3b-base," list.set(list.size() - 1, list.get(list.size() - 1) * operand.get(i + 1)); -",413,1.0 +",413,1.0 2025-09-22T13:56:55.666328,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -3459,7 +3459,7 @@ function numerical_letter_grade(grades){ return grades_array; }",220.86810000473633,,Completions generated successfully. Ready for display in your IDE.,7007e2a9-f55a-4c6d-bfae-4cf2b6b21106,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 2.3){ -",192,1.0 +",192,1.0 2025-09-22T13:56:55.805558,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -3480,7 +3480,7 @@ function even_odd_count(num){ }","} ",138.73489999969024,,Completions generated successfully. Ready for display in your IDE.,43fe4ed2-cd91-49ac-960d-2cb3d6b86f68,1,deepseek-ai/deepseek-coder-1.3b-base," return([evens, odds]); -",114,0.9814 +",114,0.9814 2025-09-22T13:56:55.893296,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -3508,7 +3508,7 @@ function triples_sum_to_zero(l){ } ",87.78199999505887,,Completions generated successfully. Ready for display in your IDE.,9ee949ec-25e7-4de9-8e25-253bae54a821,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,0.9871 +",63,0.9871 2025-09-22T13:56:56.012091,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -3585,7 +3585,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",118.5362999967765,,Completions generated successfully. Ready for display in your IDE.,45d4a86a-452f-4165-8fa9-fc130ae0d724,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:56:56.552731,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -3609,7 +3609,7 @@ function find_closest_elements(numbers){ console.log(""The smallest difference is "" + closest); console.log (""The two values with the smallest differences are:""); return [a,b].sort((a, b) => a-b); -",516,0.8487 +",516,0.8487 2025-09-22T13:56:56.647835,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -3632,7 +3632,7 @@ D.1 Recursion Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",95.33309999824269,,Completions generated successfully. Ready for display in your IDE.,7bf041c1-cd25-43e6-b79c-fb757afa130c,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",72,1.0 + return count",72,1.0 2025-09-22T13:56:56.702299,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -3653,7 +3653,7 @@ function can_arrange(arr){ return res; } ",53.89009999635164,,Completions generated successfully. Ready for display in your IDE.,aedf965c-8715-41fa-b8b9-4d5bd2d9ad16,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:56:56.813806,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -3725,7 +3725,7 @@ function minPath(grid, k){ return path; }",111.30150000099093,,Completions generated successfully. Ready for display in your IDE.,f5a97116-4aff-471b-b401-7e6fbdd172d7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:56:57.024819,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3769,7 +3769,7 @@ class Problem { } }",210.75950000522425,,Completions generated successfully. Ready for display in your IDE.,ab35df81-9f37-4f5c-b740-44cedf88653a,1,deepseek-ai/deepseek-coder-1.3b-base," while (n >= 10 || n < 0) { -",183,0.8902 +",183,0.8902 2025-09-22T13:56:57.196169,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -3781,7 +3781,7 @@ def all_prefixes(string: str) -> List[str]: return prefixes ",171.49399999470916,,Completions generated successfully. Ready for display in your IDE.,c94ccdc9-0305-409f-af3d-8fdad4f63079,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(string)): -",153,1.0 +",153,1.0 2025-09-22T13:56:57.233706,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3815,7 +3815,7 @@ class Problem { } return true; } -}",36.65739999996731,,Completions generated successfully. Ready for display in your IDE.,e14878e2-a5c9-407a-952d-2cf7fc95795e,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",36.65739999996731,,Completions generated successfully. Ready for display in your IDE.,e14878e2-a5c9-407a-952d-2cf7fc95795e,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:56:57.427735,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3846,7 +3846,7 @@ class Problem { } }",193.84499999432592,,Completions generated successfully. Ready for display in your IDE.,bf6fadee-fea6-4139-842e-5943888d5374,1,deepseek-ai/deepseek-coder-1.3b-base," if (l.get(i) < a) { -",176,1.0 +",176,1.0 2025-09-22T13:56:57.524335,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -3895,7 +3895,7 @@ function check_dict_case(dict){ return true; }",97.11059999972349,,Completions generated successfully. Ready for display in your IDE.,289b5dc1-f499-440b-91b1-7b44936fad2e,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",77,1.0 +",77,1.0 2025-09-22T13:56:57.707622,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -3917,7 +3917,7 @@ function circular_shift(x, shift){ } ",182.2602999964147,,Completions generated successfully. Ready for display in your IDE.,3a9fa82a-50bb-4f49-a76a-f78582e421c9,1,deepseek-ai/deepseek-coder-1.3b-base," //convert to String first var str = ''+x; -",165,0.903 +",165,0.903 2025-09-22T13:56:58.038415,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -3944,7 +3944,7 @@ function file_name_check(file_name){ ",330.9120000049006,,Completions generated successfully. Ready for display in your IDE.,b57a36b4-9dfb-41c2-9a1c-7a687b154c30,1,deepseek-ai/deepseek-coder-1.3b-base," let str_2 = file_name.substring(file_name.lastIndexOf('.') + 1); -",307,0.9973 +",307,0.9973 2025-09-22T13:56:58.173411,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -3993,7 +3993,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",134.94730000093114,,Completions generated successfully. Ready for display in your IDE.,313b4de3-52db-47ac-869f-d7c9a10ccb98,1,deepseek-ai/deepseek-coder-1.3b-base," while(num >= 0){ -",115,0.9896 +",115,0.9896 2025-09-22T13:56:58.521680,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4045,7 +4045,7 @@ class Problem { }",347.96889999415725,,Completions generated successfully. Ready for display in your IDE.,4fc1e92c-0d51-47d7-9bb8-d859174cc4b6,1,deepseek-ai/deepseek-coder-1.3b-base," char first = name.charAt(0); if ((first >= 'a' && first <= 'z')) { -",327,0.9726 +",327,0.9726 2025-09-22T13:56:58.688072,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -4078,7 +4078,7 @@ class Problem { ",166.0156000070856,,Completions generated successfully. Ready for display in your IDE.,253a20e7-6497-4300-bc92-7e503ae7893d,1,deepseek-ai/deepseek-coder-1.3b-base," while (j < n - 1): -",146,1.0 +",146,1.0 2025-09-22T13:56:58.781324,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -4120,7 +4120,7 @@ function is_sorted(lst){ } ",93.0519000030472,,Completions generated successfully. Ready for display in your IDE.,4be7c619-0ced-47b8-a792-f2f5ad767c21,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:58.891396,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -4187,7 +4187,7 @@ function numerical_letter_grade(grades){ return grades_array; }",109.77019999700133,,Completions generated successfully. Ready for display in your IDE.,dd5c5800-f84c-4c77-b63e-246442e66071,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:56:59.030841,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4265,7 +4265,7 @@ class Problem { } }",139.73879999684868,,Completions generated successfully. Ready for display in your IDE.,b98ac074-49c9-4dc8-bd9e-9f0d8e84bbfc,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""c""; -",112,1.0 +",112,1.0 2025-09-22T13:56:59.135918,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -4279,7 +4279,7 @@ function filter_by_substring(strings, substring){ } }",},104.86220000166213,,Completions generated successfully. Ready for display in your IDE.,9c31170b-3bf9-4838-b3bd-806a17e231c2,1,deepseek-ai/deepseek-coder-1.3b-base," return filtered; -",82,1.0 +",82,1.0 2025-09-22T13:56:59.242192,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -4302,7 +4302,7 @@ function factorize(n){ } ",105.59000000648666,,Completions generated successfully. Ready for display in your IDE.,27ee7cb8-ac8f-4df8-b0f3-fc8aa79c7e36,1,deepseek-ai/deepseek-coder-1.3b-base," factor++; -",83,1.0 +",83,1.0 2025-09-22T13:56:59.696592,[1],"from typing import Tuple def bf(planet1: str, planet2: str) -> Tuple[str, ...]: @@ -4333,7 +4333,7 @@ def bf(planet1: str, planet2: str) -> Tuple[str, ...]: else: planets = planets[end+1:start] return tuple(planets)",454.05709999613464,,Completions generated successfully. Ready for display in your IDE.,31903ab4-d785-402c-a396-56afc46961c8,1,deepseek-ai/deepseek-coder-1.3b-base," planets = ['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune'] -",428,0.9785 +",428,0.9785 2025-09-22T13:57:00.408202,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4379,7 +4379,7 @@ class Problem { if ((char)(strongest[strength].toLowerCase()) < (char)(s[strength].toLowerCase())){ strongest = s; } -",685,0.9375 +",685,0.9375 2025-09-22T13:57:00.574935,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4411,7 +4411,7 @@ class Problem { } }",166.56229999352945,,Completions generated successfully. Ready for display in your IDE.,7add9e77-0259-4eab-83ab-1e1f34b36e45,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:00.708631,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4476,7 +4476,7 @@ class Problem { } }",133.5021000049892,,Completions generated successfully. Ready for display in your IDE.,4127a570-bb4f-4cd3-9c07-8bf3be41b671,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",103,1.0 +",103,1.0 2025-09-22T13:57:00.827052,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -4492,7 +4492,7 @@ function how_many_times(string, substring){ return count; }",117.61220000335015,,Completions generated successfully. Ready for display in your IDE.,138bba3e-c874-4f2a-afe9-4799fbab56bd,1,deepseek-ai/deepseek-coder-1.3b-base," var count = 0; -",94,1.0 +",94,1.0 2025-09-22T13:57:01.528226,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -4520,7 +4520,7 @@ function is_multiply_prime(a){ for(let j=0 ;j1;j++ ) { if((Math.floor(a/prime[j])) == ((a)/prime[j]) && a%prime[j]==0){ break; - }",674,0.9329 + }",674,0.9329 2025-09-22T13:57:01.589457,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -4560,7 +4560,7 @@ function int_to_mini_roman(number){ return output; } ",61.26190000213683,,Completions generated successfully. Ready for display in your IDE.,98dea33b-bd3e-41e7-871e-e74ba6c0644d,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:01.822881,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4593,7 +4593,7 @@ class Problem { }",232.80359999625944,,Completions generated successfully. Ready for display in your IDE.,914dd7a1-c415-41e1-999c-03a3af4e98c7,1,deepseek-ai/deepseek-coder-1.3b-base," long a1 = a; long b1 = b; -",204,0.9875 +",204,0.9875 2025-09-22T13:57:01.894792,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4633,7 +4633,7 @@ class Problem { } return memo[(int) n]; }",,72.13359999877866,,Completions generated successfully. Ready for display in your IDE.,7e843589-a5ab-4a6e-b5e5-c9bdff8567c0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:02.052924,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -4691,7 +4691,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.03220000088913,,Completions generated successfully. Ready for display in your IDE.,6dd4d1cf-d64e-49a4-a297-ee0621c0aa13,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 0.0: -",132,1.0 +",132,1.0 2025-09-22T13:57:02.183529,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -4714,7 +4714,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",130.24030000087805,,Completions generated successfully. Ready for display in your IDE.,e279cfda-c28c-4e6d-bd7b-93901ed5d592,1,deepseek-ai/deepseek-coder-1.3b-base," odd_count += 1 -",108,1.0 +",108,1.0 2025-09-22T13:57:02.352132,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4738,7 +4738,7 @@ class Problem { } }",168.10630000691162,,Completions generated successfully. Ready for display in your IDE.,1897fe45-01f1-4eb3-8396-0691e1ab39aa,1,deepseek-ai/deepseek-coder-1.3b-base," al.add(st.nextToken()); -",143,1.0 +",143,1.0 2025-09-22T13:57:02.444698,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -4760,7 +4760,7 @@ function anti_shuffle(s){ result.push(temp.join(""""));"," return result.join("" ""); }",92.18479999981355,,Completions generated successfully. Ready for display in your IDE.,e85c8ea4-a3f4-4fa2-943e-e5da66c39b1a,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",68,1.0 +",68,1.0 2025-09-22T13:57:02.632430,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4788,7 +4788,7 @@ class Problem { } }",188.00690000352915,,Completions generated successfully. Ready for display in your IDE.,10666ff1-b09e-4881-8a8f-0c5ea435e7c5,1,deepseek-ai/deepseek-coder-1.3b-base," String[] words = sentence.trim().split("" ""); -",164,0.9911 +",164,0.9911 2025-09-22T13:57:02.860638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4856,7 +4856,7 @@ class Problem { } }",227.8223999965121,,Completions generated successfully. Ready for display in your IDE.,d211c322-9e16-47d3-9050-640076dc3b6e,1,deepseek-ai/deepseek-coder-1.3b-base," seen_2_in_a_row = true; -",198,1.0 +",198,1.0 2025-09-22T13:57:02.987359,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -4880,7 +4880,7 @@ class Problem { ",126.14519999624463,,Completions generated successfully. Ready for display in your IDE.,0a8f4208-1d29-4d8a-ae2e-2d51438e5001,1,deepseek-ai/deepseek-coder-1.3b-base," if count < 0: -",105,1.0 +",105,1.0 2025-09-22T13:57:03.461507,[1],"//Given a positive integer n, you have to make a pile of n levels of stones. // The first level has n stones. // The number of stones in the next level is: @@ -4899,7 +4899,7 @@ function make_a_pile(n){ // console.log(""arr"", typeof arr); const nums = Math.pow((2*i + 1), 2); arr[i]=nums; -",451,0.9334 +",451,0.9334 2025-09-22T13:57:03.569791,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -4932,7 +4932,7 @@ function make_a_pile(n){ ",108.55050000100164,,Completions generated successfully. Ready for display in your IDE.,f859ef58-7d16-4ce1-9238-45ebbf1c2433,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",84,1.0 +",84,1.0 2025-09-22T13:57:03.811959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4980,7 +4980,7 @@ class Problem { } }",241.22640000132378,,Completions generated successfully. Ready for display in your IDE.,ea6e0e93-9c8f-4da3-bc03-c32517084923,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList l_s = new ArrayList(); -",212,1.0 +",212,1.0 2025-09-22T13:57:03.961002,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -5010,7 +5010,7 @@ function closest_integer(value){ } ",149.10450000024866,,Completions generated successfully. Ready for display in your IDE.,abb1f146-f87b-4d12-b0fe-43ee5cca3ba2,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.ceil(value); -",124,1.0 +",124,1.0 2025-09-22T13:57:04.208150,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -5041,7 +5041,7 @@ function same_chars(s0, s1){ ",246.5714999998454,,Completions generated successfully. Ready for display in your IDE.,7f60751e-0878-4b2e-b510-319a7961c08e,1,deepseek-ai/deepseek-coder-1.3b-base," if(s1.indexOf(s0[i]) === -1){ -",222,1.0 +",222,1.0 2025-09-22T13:57:04.324700,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -5074,7 +5074,7 @@ function prod_signs(arr){ } } ",117.00179999752436,,Completions generated successfully. Ready for display in your IDE.,c003f44f-0f70-4f29-9cb6-23f3d731bcf5,1,deepseek-ai/deepseek-coder-1.3b-base," //your code here... -",93,0.8782 +",93,0.8782 2025-09-22T13:57:04.553368,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5122,7 +5122,7 @@ class Problem { } }",228.32370000105584,,Completions generated successfully. Ready for display in your IDE.,fb6c640f-a62e-4e8b-9159-a28d04faa565,1,deepseek-ai/deepseek-coder-1.3b-base," if (min > l_e.get(j)) { -",202,1.0 +",202,1.0 2025-09-22T13:57:04.886000,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -5148,7 +5148,7 @@ function make_palindrome(string){ ",332.54390000365674,,Completions generated successfully. Ready for display in your IDE.,0c983647-a724-4558-8405-485a78f6df69,1,deepseek-ai/deepseek-coder-1.3b-base," //find the longest postfix of supplied string that is a palindrome var i = string_length-1; -",309,0.948 +",309,0.948 2025-09-22T13:57:05.003843,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5213,7 +5213,7 @@ class Problem { } }",117.31439999857685,,Completions generated successfully. Ready for display in your IDE.,1fd51cff-1f11-4b4b-aeca-053a0b7deec3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",81,1.0 +",81,1.0 2025-09-22T13:57:05.169869,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -5240,7 +5240,7 @@ def sum_squares(lst: List[float]) -> int: total = 0 for i in lst:", return total,166.15650000312598,,Completions generated successfully. Ready for display in your IDE.,71200c59-2777-4184-8299-7c6cc7ea06a2,1,deepseek-ai/deepseek-coder-1.3b-base," total += (i ** 2) -",135,0.9583 +",135,0.9583 2025-09-22T13:57:05.847614,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -5270,7 +5270,7 @@ function is_simple_power(x, n){ ",677.3353999960818,,Completions generated successfully. Ready for display in your IDE.,255a412d-201e-452b-9ae0-2a62f59f5de1,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.pow(n, Math.floor(Math.log(x)/Math.log(n))) === x && Number.isInteger(Math.pow(n, Math.floor(Math.log(x)/Math.log(n))))); { -",654,0.9607 +",654,0.9607 2025-09-22T13:57:06.018631,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -5289,7 +5289,7 @@ function is_bored(S){ } ",170.21540000132518,,Completions generated successfully. Ready for display in your IDE.,42dc7758-c2ce-4258-bd26-abf071f280ef,1,deepseek-ai/deepseek-coder-1.3b-base," if (/^i/.test(line)){ -",147,0.9829 +",147,0.9829 2025-09-22T13:57:06.468386,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -5313,7 +5313,7 @@ function anti_shuffle(s){ }",450.1712999990559,,Completions generated successfully. Ready for display in your IDE.,645fdf0d-c806-4245-9ed0-82360bac10d6,1,deepseek-ai/deepseek-coder-1.3b-base," if (!/^[a-zA-Z]+$/.test(s)) {return false;} // Split into array with each element being one word -",425,0.8954 +",425,0.8954 2025-09-22T13:57:06.572442,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5351,7 +5351,7 @@ class Problem { return lst2; }",},104.00600000139093,,Completions generated successfully. Ready for display in your IDE.,a7efe2fc-e2c6-4306-a571-0285529c41b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T13:57:06.653125,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -5372,7 +5372,7 @@ function even_odd_count(num){ } ",79.53059999999823,,Completions generated successfully. Ready for display in your IDE.,2546b0f9-7486-40b4-9718-36a2a53cccf5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T13:57:06.913312,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -5449,7 +5449,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",260.11100000323495,,Completions generated successfully. Ready for display in your IDE.,eaaf46af-3f99-41be-91b6-7bd71ae33b92,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, int) and isinstance(b, float): -",232,1.0 +",232,1.0 2025-09-22T13:57:07.252859,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5477,7 +5477,7 @@ class Problem { }",339.71139999630395,,Completions generated successfully. Ready for display in your IDE.,d10647e8-859b-45b6-81d7-12cafa74612c,1,deepseek-ai/deepseek-coder-1.3b-base," System.out.println((long)(bd.doubleValue())); return Math.round(bd.doubleValue()); -",315,0.9443 +",315,0.9443 2025-09-22T13:57:07.371373,[1],"def fizz_buzz(n: int) -> int: """"""Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. >>> fizz_buzz(50) @@ -5492,7 +5492,7 @@ class Problem { if i % 11 == 0 or i % 13 == 0:"," return x ",117.87190000177361,,Completions generated successfully. Ready for display in your IDE.,585b5095-cd95-4d93-a8bf-fab4b1a198db,1,deepseek-ai/deepseek-coder-1.3b-base," x += 1 -",92,1.0 +",92,1.0 2025-09-22T13:57:07.439679,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -5519,7 +5519,7 @@ function pairs_sum_to_zero(l){ i = i + 1 } return false",,68.66770000488032,,Completions generated successfully. Ready for display in your IDE.,dcd8658f-8ffa-4866-9333-9c6ab460f0fd,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T13:57:07.589592,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -5546,7 +5546,7 @@ def find_max(words: List[str]) -> str: return max_word ",149.39049999520648,,Completions generated successfully. Ready for display in your IDE.,ab72caef-7f1c-48a2-89b2-4fb35376f949,1,deepseek-ai/deepseek-coder-1.3b-base," max_num = unique_num -",125,1.0 +",125,1.0 2025-09-22T13:57:07.752972,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -5572,7 +5572,7 @@ function string_xor(a, b){ return result; }",163.73449999809964,,Completions generated successfully. Ready for display in your IDE.,87a8256d-bb6e-462e-8221-547c769f6d0e,1,deepseek-ai/deepseek-coder-1.3b-base," arrayXOR.push(1); -",133,1.0 +",133,1.0 2025-09-22T13:57:07.844227,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5595,7 +5595,7 @@ class Problem { } return answer;",},90.55989999615122,,Completions generated successfully. Ready for display in your IDE.,1d6b8604-e145-4909-b4ec-2a0845f68d96,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:08.563160,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -5680,7 +5680,7 @@ if __name__ == ""__main__"": import doctest doctest.testmod() -Trying",693,0.998 +Trying",693,0.998 2025-09-22T13:57:08.656794,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5720,7 +5720,7 @@ class Problem { } return memo[(int) n];",},93.57789999921806,,Completions generated successfully. Ready for display in your IDE.,3c490ffe-48a5-4477-aadb-3d80e65388c4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:08.979095,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -5746,7 +5746,7 @@ class Problem { return word[i] return '' ",321.7603999946732,,Completions generated successfully. Ready for display in your IDE.,4d5f143e-e7d5-44e6-9ce8-3d222d0d848e,1,deepseek-ai/deepseek-coder-1.3b-base," if len(word) < 3 or set(word) & set('aeiouAEIOU') == {}: -",297,0.9723 +",297,0.9723 2025-09-22T13:57:09.049164,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5775,7 +5775,7 @@ class Problem { } return (sqrt * 2) % p; }",,69.8095000043395,,Completions generated successfully. Ready for display in your IDE.,4e255e07-8fa4-4f74-bb99-8e8dec40c6c1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T13:57:09.155606,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5809,7 +5809,7 @@ class Problem { } }",106.08680000586901,,Completions generated successfully. Ready for display in your IDE.,a39fee58-0655-4687-a6f2-01496ca4e86d,1,deepseek-ai/deepseek-coder-1.3b-base," open++; -",83,1.0 +",83,1.0 2025-09-22T13:57:09.333343,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5851,7 +5851,7 @@ class Problem { } }",177.9549999992014,,Completions generated successfully. Ready for display in your IDE.,afa2d577-d665-442a-940f-374d02306f43,1,deepseek-ai/deepseek-coder-1.3b-base," if (c1 < 0) break; -",152,1.0 +",152,1.0 2025-09-22T13:57:09.623819,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5887,7 +5887,7 @@ class Problem { } }",290.3502000044682,,Completions generated successfully. Ready for display in your IDE.,2603564c-022b-41a3-a333-db002fb7300f,1,deepseek-ai/deepseek-coder-1.3b-base," for (int k = j + 1; k < arrayList.size(); k++) { -",265,1.0 +",265,1.0 2025-09-22T13:57:09.781080,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5924,7 +5924,7 @@ class Problem { } }",157.28390000003856,,Completions generated successfully. Ready for display in your IDE.,97638c4b-e5dc-4096-be30-03fa42114af7,1,deepseek-ai/deepseek-coder-1.3b-base," open_paren++; -",132,1.0 +",132,1.0 2025-09-22T13:57:09.996811,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5977,7 +5977,7 @@ class Problem { } }",215.35340000264114,,Completions generated successfully. Ready for display in your IDE.,911decbd-3db4-4489-9522-e12fc241b608,1,deepseek-ai/deepseek-coder-1.3b-base," newArr.add((long)minValue); -",184,0.9495 +",184,0.9495 2025-09-22T13:57:10.085341,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6005,7 +6005,7 @@ class Problem { } }",87.58039999520406,,Completions generated successfully. Ready for display in your IDE.,e46dbe42-6307-4ad0-a66b-d378efef26a3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:10.262141,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6043,7 +6043,7 @@ class Problem { } }",177.5119000012637,,Completions generated successfully. Ready for display in your IDE.,628e5e30-73a5-429a-91ba-bfcb67b446d3,1,deepseek-ai/deepseek-coder-1.3b-base," res.add((long)4l); -",151,0.967 +",151,0.967 2025-09-22T13:57:10.354675,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6069,7 +6069,7 @@ class Problem { } }",91.63819999957923,,Completions generated successfully. Ready for display in your IDE.,8fa723e3-0ce0-4bb1-b058-045d2e22db73,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:10.470137,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -6102,7 +6102,7 @@ function digits(n){ ",114.98140000185231,,Completions generated successfully. Ready for display in your IDE.,ababbb5c-406b-46db-993c-3de30cef94dd,1,deepseek-ai/deepseek-coder-1.3b-base," return 0; -",93,1.0 +",93,1.0 2025-09-22T13:57:11.171840,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -6134,7 +6134,7 @@ function is_multiply_prime(a){ for(var j=0;j int: """""" Create a function that takes a value (string) representing a number @@ -6159,7 +6159,7 @@ function is_multiply_prime(a){ if abs(int(value)-float(value)) < 0.5: return integer_val else: - return integer_val+1 if ((abs(int(value)-float(value)) > 0.5) and (value >= 0)) else integer_val-",670,0.9465 + return integer_val+1 if ((abs(int(value)-float(value)) > 0.5) and (value >= 0)) else integer_val-",670,0.9465 2025-09-22T13:57:12.281348,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -6198,7 +6198,7 @@ function valid_date(date){ } ",416.6438000029302,,Completions generated successfully. Ready for display in your IDE.,c57ce04e-3082-46f0-913e-124bbd587bd0,1,deepseek-ai/deepseek-coder-1.3b-base," }else if((month == 4 || month == 6 || month == 9 || month == 11) && (day < 1 || day > 30)){ -",389,0.9906 +",389,0.9906 2025-09-22T13:57:12.408352,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6230,7 +6230,7 @@ class Problem { }"," } }",126.27779999456834,,Completions generated successfully. Ready for display in your IDE.,4e95f963-f33f-45fd-8a2d-c796cf7b3ab2,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",97,1.0 +",97,1.0 2025-09-22T13:57:12.733563,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -6251,7 +6251,7 @@ function sort_array(arr){ }); }",325.03279999946244,,Completions generated successfully. Ready for display in your IDE.,f1647dae-1b1f-47b7-98f0-d1a2b32e7d92,1,deepseek-ai/deepseek-coder-1.3b-base," let bBinary = b.toString(2).split('').filter(el => el === '1').length; -",299,1.0 +",299,1.0 2025-09-22T13:57:12.817426,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6278,7 +6278,7 @@ class Problem { } }",83.6866999961785,,Completions generated successfully. Ready for display in your IDE.,fff320bf-bf8d-407f-b894-eb60becbc345,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T13:57:12.895544,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6307,7 +6307,7 @@ class Problem { long oranges = Long.parseLong(splitted[3]); return n - apples - oranges; }",,77.93690000107745,,Completions generated successfully. Ready for display in your IDE.,1a0f950b-a23f-4e87-9b50-79bf3624a1e6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:57:13.021171,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -6338,7 +6338,7 @@ function skjkasdkd(lst){ } return sum; }",125.56960000074469,,Completions generated successfully. Ready for display in your IDE.,83a0739d-d89a-4f46-8906-fce910114f31,1,deepseek-ai/deepseek-coder-1.3b-base," function prime(n){ -",101,1.0 +",101,1.0 2025-09-22T13:57:13.125404,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6376,7 +6376,7 @@ class Problem { } }",104.14669999590842,,Completions generated successfully. Ready for display in your IDE.,1d1b06e6-b304-44a5-8587-39351b08bd11,1,deepseek-ai/deepseek-coder-1.3b-base," } -",79,1.0 +",79,1.0 2025-09-22T13:57:13.212727,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -6406,7 +6406,7 @@ function closest_integer(value){ } ",87.50240000517806,,Completions generated successfully. Ready for display in your IDE.,77faac8d-fe4c-4c45-81ce-f69443f4dc56,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:13.301742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6434,7 +6434,7 @@ class Problem { } return p;",},88.52379999734694,,Completions generated successfully. Ready for display in your IDE.,deccfacc-d3eb-4693-ac25-8b536a5717b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:13.459532,[1],"from typing import List def pairs_sum_to_zero(l: List[int]) -> bool: @@ -6458,7 +6458,7 @@ def pairs_sum_to_zero(l: List[int]) -> bool: return False ",157.1637999950326,,Completions generated successfully. Ready for display in your IDE.,3f894919-0c3d-4a73-9c39-c507d9d159af,1,deepseek-ai/deepseek-coder-1.3b-base," if (x + y == 0): -",134,0.8809 +",134,0.8809 2025-09-22T13:57:13.504909,[1],"def remove_vowels(text: str) -> str: """""" remove_vowels is a function that takes string and returns string without vowels. @@ -6474,7 +6474,7 @@ def pairs_sum_to_zero(l: List[int]) -> bool: 'zbcd' """""" "," return ''.join(filter(lambda x: x not in vowels, text)) -",45.010699999693315,,Completions generated successfully. Ready for display in your IDE.,c60aa39c-0099-433b-a535-8ed0cbbd9b65,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",45.010699999693315,,Completions generated successfully. Ready for display in your IDE.,c60aa39c-0099-433b-a535-8ed0cbbd9b65,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:13.633225,[1],"//Given a string 'text', return its md5 hash equivalent string. // If 'text' is an empty string, return undefined. // >>> string_to_md5(""Hello world"") @@ -6485,7 +6485,7 @@ function string_to_md5(text){ } ",128.4534000005806,,Completions generated successfully. Ready for display in your IDE.,77892d9e-edaa-4872-9da4-67a8061644a1,1,deepseek-ai/deepseek-coder-1.3b-base," console.log('undefined'); -",107,0.9016 +",107,0.9016 2025-09-22T13:57:13.686653,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -6511,7 +6511,7 @@ def find_max(words: List[str]) -> str: max_word = word return max_word ",53.40240000077756,,Completions generated successfully. Ready for display in your IDE.,23c5794b-2d60-4105-9741-feeb3bae61db,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:57:13.816257,[1],"def fruit_distribution(s: str, n: int) -> int: """""" In this task, you will be given a string that represents a number of apples and oranges @@ -6538,7 +6538,7 @@ def find_max(words: List[str]) -> str: ",129.4449999986682,,Completions generated successfully. Ready for display in your IDE.,f760b7f9-7eaa-4506-9c14-b53796f51a3e,1,deepseek-ai/deepseek-coder-1.3b-base," except ValueError: -",99,1.0 +",99,1.0 2025-09-22T13:57:13.934180,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -6577,7 +6577,7 @@ function valid_date(date){ } ",117.72999999811873,,Completions generated successfully. Ready for display in your IDE.,faa514c0-3951-44ba-9c9a-960ba8e65604,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",90,1.0 +",90,1.0 2025-09-22T13:57:14.369409,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -6593,7 +6593,7 @@ function filter_by_substring(strings, substring){ //loop through the string and check each one against our search term for (var i = 0 ; i < strings.length; ++i) { -",412,0.9216 +",412,0.9216 2025-09-22T13:57:14.744254,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -6615,7 +6615,7 @@ function cycpattern_check(a, b){ } ",374.3818000002648,,Completions generated successfully. Ready for display in your IDE.,94723c98-8f64-4d54-bde1-b2936f0f866b,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0 ;i < Math.floor((Math.sqrt(a.length))/3)*4 + 5;) { -",351,0.8684 +",351,0.8684 2025-09-22T13:57:15.034347,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6647,7 +6647,7 @@ class Problem { return false; } }",289.7977999964496,,Completions generated successfully. Ready for display in your IDE.,b7a62009-d2af-4f7f-b48b-bc609e617ed7,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < l.size() - 1; i++) { -",265,0.9966 +",265,0.9966 2025-09-22T13:57:15.186704,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6679,7 +6679,7 @@ class Problem { } }",152.334599995811,,Completions generated successfully. Ready for display in your IDE.,62e4bd91-e966-4d1c-a79e-465e6d5522d1,1,deepseek-ai/deepseek-coder-1.3b-base," long max_depth = 0; -",126,0.9158 +",126,0.9158 2025-09-22T13:57:15.299286,[1],"def strlen(string: str) -> int: """""" Return length of given string >>> strlen('') @@ -6692,7 +6692,7 @@ class Problem { ",112.45220000273548,,Completions generated successfully. Ready for display in your IDE.,527b0710-12a6-4fef-905a-8c269429325e,1,deepseek-ai/deepseek-coder-1.3b-base," length += 1 -",91,1.0 +",91,1.0 2025-09-22T13:57:15.419617,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -6769,7 +6769,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",119.57280000206083,,Completions generated successfully. Ready for display in your IDE.,13dd0e4d-b814-4c09-bce0-c8d7356f0013,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",88,1.0 +",88,1.0 2025-09-22T13:57:15.536682,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6820,7 +6820,7 @@ class Problem { } }",116.76450000231853,,Completions generated successfully. Ready for display in your IDE.,c90a2f80-f0ec-4400-a1d0-f2195194c6b7,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",89,1.0 +",89,1.0 2025-09-22T13:57:15.600916,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -6844,7 +6844,7 @@ function reverse_delete(s, c){ result.push(filterString, false); } return result;",,64.54600000142818,,Completions generated successfully. Ready for display in your IDE.,5583d8ef-e993-409a-b93a-a789cbdb59aa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:57:15.694800,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6895,7 +6895,7 @@ class Problem { } }",93.47400000115158,,Completions generated successfully. Ready for display in your IDE.,084752ae-03fd-4510-89f5-8f8964874633,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:15.868884,[1],"from typing import List def sum_squares(lst: List[int]) -> int: @@ -6917,7 +6917,7 @@ def sum_squares(lst: List[int]) -> int: lst[index] = lst[index] ** 2"," lst[index] = lst[index] ** 3 return sum(lst)",173.9201999953366,,Completions generated successfully. Ready for display in your IDE.,aba7c6ec-7be7-452b-9de4-4819ccb68749,1,deepseek-ai/deepseek-coder-1.3b-base," elif index % 4 != 0: -",150,0.9753 +",150,0.9753 2025-09-22T13:57:15.956244,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -6948,7 +6948,7 @@ function total_match(lst1, lst2){ } ",87.34540000295965,,Completions generated successfully. Ready for display in your IDE.,e4b67de6-6754-4263-a43b-a32e68c53e7a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:16.070605,[1],"from typing import List def max_fill(grid: List[List[int]], capacity: int) -> int: @@ -6985,7 +6985,7 @@ def max_fill(grid: List[List[int]], capacity: int) -> int: if 1 in row[i:i+capacity]: times += 1",,114.2318999991403,,Completions generated successfully. Ready for display in your IDE.,b89e1fdc-a136-4d2b-82c9-0e0b2716ed2a,1,deepseek-ai/deepseek-coder-1.3b-base," - return times",90,0.9921 + return times",90,0.9921 2025-09-22T13:57:16.165397,[1],"from typing import Optional def string_to_md5(text: str) -> Optional[str]: @@ -7001,7 +7001,7 @@ def string_to_md5(text: str) -> Optional[str]: return None"," return hashlib.md5(text.encode('utf-8')).hexdigest() ",94.16659999988042,,Completions generated successfully. Ready for display in your IDE.,4e7238d9-4ce7-474b-ab97-f7d1549233e1,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:57:16.239178,[1],"//Create a function that returns an array (a, b), where 'a' is // the largest of negative integers, and 'b' is the smallest // of positive integers in an array. @@ -7015,7 +7015,7 @@ def string_to_md5(text: str) -> Optional[str]: // [undefined, undefined] function largest_smallest_integers(lst){ return[lst.filter(x=>x<0).sort((a,b)=>b-a)[0],lst.filter(x=>x>0).sort((a,b)=>a-b)[0]]",,73.853699999745,,Completions generated successfully. Ready for display in your IDE.,5ec41d61-060c-4ceb-8f32-823aa638327b,1,deepseek-ai/deepseek-coder-1.3b-base,"; -}",52,1.0 +}",52,1.0 2025-09-22T13:57:16.323490,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -7036,7 +7036,7 @@ function even_odd_count(num){ } ",83.82540000457084,,Completions generated successfully. Ready for display in your IDE.,12d74332-5615-4260-a0e4-d3acccd1acb3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:16.649248,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -7075,7 +7075,7 @@ function find_max(words){ return max_unique_word; }",325.37479999882635,,Completions generated successfully. Ready for display in your IDE.,065dc21a-902f-4752-9fc9-8769d542ff55,1,deepseek-ai/deepseek-coder-1.3b-base," else if(Object.keys(unique_word).length > Object.keys(max_unique).length){ -",298,1.0 +",298,1.0 2025-09-22T13:57:16.825574,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7153,7 +7153,7 @@ class Problem { } }",176.62779999955092,,Completions generated successfully. Ready for display in your IDE.,c99968ce-534d-4f5f-9306-82331e3e6141,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 40) { -",144,1.0 +",144,1.0 2025-09-22T13:57:17.256635,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -7184,7 +7184,7 @@ function will_it_fly(q, w){ const element = q[i] * -1; // console.log(""element"", element); if(!will_it_fly(element)){ -",405,0.8964 +",405,0.8964 2025-09-22T13:57:17.397733,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7214,7 +7214,7 @@ class Problem { } }",140.9568000017316,,Completions generated successfully. Ready for display in your IDE.,0201adec-8b2b-4bcd-8cc9-36a21f955854,1,deepseek-ai/deepseek-coder-1.3b-base," if (n > 1) -",115,1.0 +",115,1.0 2025-09-22T13:57:17.469341,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7262,7 +7262,7 @@ class Problem { return valid_date; } }",72.06720000249334,,Completions generated successfully. Ready for display in your IDE.,4c1cf7ec-276b-4e5d-84d2-ed0321d624e4,1,deepseek-ai/deepseek-coder-1.3b-base," -",45,1.0 +",45,1.0 2025-09-22T13:57:17.559872,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7302,7 +7302,7 @@ class Problem { } return arr.size() == 3;",},89.57910000026459,,Completions generated successfully. Ready for display in your IDE.,42c88fa0-70a4-4479-a8b3-ebdac8406bd4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:17.799686,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -7330,7 +7330,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: if n_uppercase - n_lowercase > best_strength:"," best_extension = extension return class_name + '.' + best_extension",239.88240000471706,,Completions generated successfully. Ready for display in your IDE.,44b87d43-8455-4187-8bcb-be37c7f93277,1,deepseek-ai/deepseek-coder-1.3b-base," best_strength = n_uppercase - n_lowercase -",214,1.0 +",214,1.0 2025-09-22T13:57:17.971772,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -7369,7 +7369,7 @@ function pluck(arr){ ",171.90010000194889,,Completions generated successfully. Ready for display in your IDE.,1e980af1-eb5d-48b6-bcdd-6156e997aa72,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ continue; } - ",140,0.9549 + ",140,0.9549 2025-09-22T13:57:18.236155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7401,7 +7401,7 @@ class Problem { } }",263.76059999893187,,Completions generated successfully. Ready for display in your IDE.,90a01308-64eb-4045-8adb-7acaddba15e8,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < game.size(); ++i) { -",238,1.0 +",238,1.0 2025-09-22T13:57:18.326928,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7439,7 +7439,7 @@ class Problem { } }",90.96979999594623,,Completions generated successfully. Ready for display in your IDE.,d96c2360-1b6e-4dbc-acb3-1a1fe4030d31,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:57:18.682137,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -7467,7 +7467,7 @@ function strange_sort_list(lst){ }",355.04930000024615,,Completions generated successfully. Ready for display in your IDE.,212c2b38-7779-4cb8-9eca-21cc65c2e24f,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < Math.floor(lst.length / 2); ++i ) { -",329,0.9856 +",329,0.9856 2025-09-22T13:57:18.789292,[1],"from typing import List def factorize(n: int) -> List[int]: @@ -7489,7 +7489,7 @@ def factorize(n: int) -> List[int]: n = n / i"," return factors ",106.8479000023217,,Completions generated successfully. Ready for display in your IDE.,43e892e2-2074-45e7-9f14-4fa49439c539,1,deepseek-ai/deepseek-coder-1.3b-base," i += 1 -",84,1.0 +",84,1.0 2025-09-22T13:57:18.879518,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7513,7 +7513,7 @@ class Problem { } }",90.00750000268454,,Completions generated successfully. Ready for display in your IDE.,07f53544-02a3-4937-8d2c-253595f475e4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:19.058221,[1],"def is_prime(n: int) -> bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -7539,7 +7539,7 @@ class Problem { ",178.34160000347765,,Completions generated successfully. Ready for display in your IDE.,c3b0e34d-47db-46f2-a315-ece27368cb07,1,deepseek-ai/deepseek-coder-1.3b-base," # Your code here if n <= 1: -",155,0.947 +",155,0.947 2025-09-22T13:57:19.681965,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -7567,7 +7567,7 @@ function make_palindrome(string){ if (s.slice(-j) != s.slice(0, j)) return false; } return true; -",600,0.9819 +",600,0.9819 2025-09-22T13:57:19.822051,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -7592,7 +7592,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",139.6530999991228,,Completions generated successfully. Ready for display in your IDE.,dfc966d6-2d21-4e26-b832-7f056ba0be88,1,deepseek-ai/deepseek-coder-1.3b-base," open_count -= 1 -",114,1.0 +",114,1.0 2025-09-22T13:57:20.517433,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7615,7 +7615,7 @@ class Problem { long max = numbers.get(0); List results = new ArrayList<>() {{ add(max); }}; for (int i = 1; i < numbers.size(); i++) { - Long current = numbers",670,0.9821 + Long current = numbers",670,0.9821 2025-09-22T13:57:20.720826,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7641,7 +7641,7 @@ class Problem { return x * y; } }",203.06540000456152,,Completions generated successfully. Ready for display in your IDE.,9303a6ff-2442-41c1-a4a9-b33a6f96fb9d,1,deepseek-ai/deepseek-coder-1.3b-base," long x = Math.abs(a % 10); -",178,1.0 +",178,1.0 2025-09-22T13:57:20.889999,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7665,7 +7665,7 @@ class Problem { return result; } }",169.1168999968795,,Completions generated successfully. Ready for display in your IDE.,07dbe39b-80fc-4d0f-b061-9a9aa1549e1e,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList result = new ArrayList<>(); -",145,0.9525 +",145,0.9525 2025-09-22T13:57:21.037915,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -7686,7 +7686,7 @@ function sort_array(arr){ }); }",147.50220000132686,,Completions generated successfully. Ready for display in your IDE.,29ec0c76-fe94-4c31-997c-2f7f6fe3ee20,1,deepseek-ai/deepseek-coder-1.3b-base," return aBinary - bBinary; -",123,1.0 +",123,1.0 2025-09-22T13:57:21.301258,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -7714,7 +7714,7 @@ function get_max_triples(n){ } ",263.05379999394063,,Completions generated successfully. Ready for display in your IDE.,4735bd1a-1afe-4a08-bdb9-731d3737f3ff,1,deepseek-ai/deepseek-coder-1.3b-base," for(let k = j+1; k(Arrays.asList(total, left));",},105.69789999863133,,Completions generated successfully. Ready for display in your IDE.,9fcf8146-ca12-409c-a66b-14bf99885f83,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:21.588407,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -7784,7 +7784,7 @@ function get_row(lst, x){ } ",180.92579999938607,,Completions generated successfully. Ready for display in your IDE.,70197b38-46ba-45eb-b74b-3c662bf922a2,1,deepseek-ai/deepseek-coder-1.3b-base," if (array[j] === x){ -",155,0.9118 +",155,0.9118 2025-09-22T13:57:21.667277,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7823,7 +7823,7 @@ class Problem { } return result; }",,79.16040000418434,,Completions generated successfully. Ready for display in your IDE.,d3240523-e173-4d58-9eb3-a69974d394ff,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:21.724566,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -7849,7 +7849,7 @@ def by_length(arr: List[int]) -> List[str]: digits[i] = ['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight', 'Nine'][digits[i] - 1] return digits ",56.585899998026434,,Completions generated successfully. Ready for display in your IDE.,143540bf-f7d0-418f-8557-80facd25326f,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:57:21.906987,[1],"def anti_shuffle(s: str) -> str: """""" Write a function that takes a string and returns an ordered version of it. @@ -7869,7 +7869,7 @@ def by_length(arr: List[int]) -> List[str]: words = s.split(' ') shuffled_words = [''.join(sorted(word)) for word in words]"," ",182.41560000024037,,Completions generated successfully. Ready for display in your IDE.,91e2c29e-67ad-491b-9b6a-a30f4d04e291,1,deepseek-ai/deepseek-coder-1.3b-base," - return ' '.join(shuffled_words)",155,1.0 + return ' '.join(shuffled_words)",155,1.0 2025-09-22T13:57:22.083074,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7899,7 +7899,7 @@ class Problem { } }",176.23720000119647,,Completions generated successfully. Ready for display in your IDE.,956214a6-e1a8-42df-9f38-70dbf3a5c9ab,1,deepseek-ai/deepseek-coder-1.3b-base," while (n % d == 0) { -",151,0.9315 +",151,0.9315 2025-09-22T13:57:22.280893,[1],"from typing import List def max_element(l: List[int]) -> int: @@ -7914,7 +7914,7 @@ def max_element(l: List[int]) -> int: return max_elem ",197.2581000009086,,Completions generated successfully. Ready for display in your IDE.,7a5742bc-3898-463b-a0b9-e1ce471c1c47,1,deepseek-ai/deepseek-coder-1.3b-base," if (l[i] > max_elem): -",164,0.9629 +",164,0.9629 2025-09-22T13:57:22.479270,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -7929,7 +7929,7 @@ function flip_case(string){ } ",198.0353999970248,,Completions generated successfully. Ready for display in your IDE.,05133b86-729a-4a47-af14-2810703d2705,1,deepseek-ai/deepseek-coder-1.3b-base," result += string[i].toUpperCase(); -",175,1.0 +",175,1.0 2025-09-22T13:57:22.543733,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -7947,7 +7947,7 @@ function is_bored(S){ } } return boredoms;",,64.69030000152998,,Completions generated successfully. Ready for display in your IDE.,b205fbb5-2b42-4f90-99e1-afdb20a2ae9a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T13:57:22.832050,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -7961,7 +7961,7 @@ function largest_divisor(n){ return ld; }",287.87200000078883,,Completions generated successfully. Ready for display in your IDE.,7bd80a88-378b-4f7b-a3ef-fdc3a7e62a53,1,deepseek-ai/deepseek-coder-1.3b-base," for (let i=2;i>> prime_fib(1) // 2 @@ -8048,7 +8048,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",143.59269999840762,,Completions generated successfully. Ready for display in your IDE.,db89b77a-c207-47d1-885b-626f2509836d,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.abs(b); -",120,1.0 +",120,1.0 2025-09-22T13:57:23.155790,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -8125,7 +8125,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",114.05910000030417,,Completions generated successfully. Ready for display in your IDE.,cac2990f-f813-4819-a23f-aeb6cfaa7c3a,1,deepseek-ai/deepseek-coder-1.3b-base," return b -",87,1.0 +",87,1.0 2025-09-22T13:57:23.249163,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -8140,7 +8140,7 @@ function flip_case(string){ return result; } ",93.14150000136578,,Completions generated successfully. Ready for display in your IDE.,f89a4c74-d95d-497d-bae0-dc82e99dc029,1,deepseek-ai/deepseek-coder-1.3b-base," var result = """"; -",72,1.0 +",72,1.0 2025-09-22T13:57:23.396203,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -8165,7 +8165,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",146.82750000065425,,Completions generated successfully. Ready for display in your IDE.,7f5097b9-6b9e-4547-a4a1-1eb159a78774,1,deepseek-ai/deepseek-coder-1.3b-base," if open_count == 0: -",124,1.0 +",124,1.0 2025-09-22T13:57:23.571654,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8205,7 +8205,7 @@ class Problem { return b; } }",175.8339000007254,,Completions generated successfully. Ready for display in your IDE.,5be29487-f7c6-4f12-868c-f15e27b7066d,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList b = new ArrayList<>(); -",150,1.0 +",150,1.0 2025-09-22T13:57:23.666620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8247,7 +8247,7 @@ class Problem { } }",94.18230000301264,,Completions generated successfully. Ready for display in your IDE.,98ed9404-ee7b-4136-8a9e-7e0cbc585f62,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:24.220317,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8284,7 +8284,7 @@ class Problem { } }",554.0133999966201,,Completions generated successfully. Ready for display in your IDE.,cfd2148f-79f8-49ca-a5ad-3d977134c7b0,1,deepseek-ai/deepseek-coder-1.3b-base," if (txt.substring(len - 3, len - 2).matches("" "") && !txt.substring(len - 4, len - 3).matches(""\\w+"")) { -",529,0.9883 +",529,0.9883 2025-09-22T13:57:24.322721,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8316,7 +8316,7 @@ class Problem { } }",102.2923000055016,,Completions generated successfully. Ready for display in your IDE.,65eb812a-c9db-4891-9ccc-45810b0c1dc4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,0.9887 +",76,0.9887 2025-09-22T13:57:24.422967,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -8353,7 +8353,7 @@ function move_one_ball(arr){ } }",99.82389999640873,,Completions generated successfully. Ready for display in your IDE.,da934950-2848-409b-8172-1a12a476f242,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,0.9628 +",70,0.9628 2025-09-22T13:57:24.517003,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -8391,7 +8391,7 @@ function get_closest_vowel(word){ return result[0]; }",},94.31779999431456,,Completions generated successfully. Ready for display in your IDE.,ad19c51f-6dee-43b4-8952-30dfcad67f69,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:24.597009,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -8410,7 +8410,7 @@ function is_bored(S){ } ",78.9260999954422,,Completions generated successfully. Ready for display in your IDE.,50ba5dc6-41af-41b6-9e55-fa5dc4667bb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",57,0.9862 +",57,0.9862 2025-09-22T13:57:24.785652,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -8434,7 +8434,7 @@ Given a positive integer n, write a recursive function that returns the sum of t positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",188.50559999555117,,Completions generated successfully. Ready for display in your IDE.,f636dd00-643e-4542-b26c-a3b8c6bb6e0b,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(s)): -",151,1.0 +",151,1.0 2025-09-22T13:57:24.854004,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -8501,7 +8501,7 @@ function do_algebra(operator, operand){ return answer; } ",68.10129999939818,,Completions generated successfully. Ready for display in your IDE.,45de48c0-8bef-4903-945d-48ada8ccc2eb,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T13:57:24.923076,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -8528,7 +8528,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: min2 = lst[i] return min2 if min1 != min2 else None ",69.30339999962598,,Completions generated successfully. Ready for display in your IDE.,7698b5d6-db52-443e-b053-76459a54d7fd,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:57:25.069294,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -8549,7 +8549,7 @@ function circular_shift(x, shift){ } ",145.83539999875939,,Completions generated successfully. Ready for display in your IDE.,377f4f89-d7db-4d80-90ab-3b908f7144a5,1,deepseek-ai/deepseek-coder-1.3b-base," var len = str.length; -",115,1.0 +",115,1.0 2025-09-22T13:57:25.155179,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8578,7 +8578,7 @@ class Problem { } return b.toString();",},85.76210000319406,,Completions generated successfully. Ready for display in your IDE.,b4a5fce1-4f93-42f5-a8b1-e2b4eef17655,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:25.237980,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -8615,7 +8615,7 @@ function max_fill(grid, capacity){ } ",82.4577999956091,,Completions generated successfully. Ready for display in your IDE.,9f7dd934-a512-497c-99c5-0e991a128aa7,1,deepseek-ai/deepseek-coder-1.3b-base," -",57,1.0 +",57,1.0 2025-09-22T13:57:25.323446,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -8644,7 +8644,7 @@ function is_equal_to_sum_even(n){ } }",85.70980000513373,,Completions generated successfully. Ready for display in your IDE.,d615edfc-a9b7-4a49-93d9-80d1f8b3345c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:25.434455,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -8667,7 +8667,7 @@ function encode(message){ result += letter.toUpperCase() === letter ? letter.toLowerCase() : letter.toUpperCase(); }",},110.33820000011474,,Completions generated successfully. Ready for display in your IDE.,2a55495d-4e56-4075-9706-4ed63ee03a96,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",85,0.8848 +",85,0.8848 2025-09-22T13:57:25.562812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8709,7 +8709,7 @@ class Problem { } }",128.34419999853708,,Completions generated successfully. Ready for display in your IDE.,68d33548-7084-4bd2-966d-0298c8157016,1,deepseek-ai/deepseek-coder-1.3b-base," String strongest = """"; -",100,1.0 +",100,1.0 2025-09-22T13:57:25.813028,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -8731,7 +8731,7 @@ function fizz_buzz(n){ } ",249.63170000410173,,Completions generated successfully. Ready for display in your IDE.,22efff65-ca5d-4d3e-aa87-2a3a15e20752,1,deepseek-ai/deepseek-coder-1.3b-base," for (var j=0; j 0) { -",128,0.9712 +",128,0.9712 2025-09-22T13:57:26.138097,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8841,7 +8841,7 @@ class Problem { }",159.4175999998697,,Completions generated successfully. Ready for display in your IDE.,b3c60469-299f-4dc2-aa9b-c6764c99d7c5,1,deepseek-ai/deepseek-coder-1.3b-base," { return null; } -",134,0.9615 +",134,0.9615 2025-09-22T13:57:26.295981,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -8864,7 +8864,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",157.66230000008363,,Completions generated successfully. Ready for display in your IDE.,62241295-90ee-4e50-b2fd-55a936850f22,1,deepseek-ai/deepseek-coder-1.3b-base," if digit % 2 == 0: -",134,1.0 +",134,1.0 2025-09-22T13:57:26.374527,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8906,7 +8906,7 @@ class Problem { } return class_name + ""."" + strongest; }",,78.19290000043111,,Completions generated successfully. Ready for display in your IDE.,857377f4-c8f4-487c-bd02-39dc6b213411,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:26.436772,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -8943,7 +8943,7 @@ function even_odd_palindrome(n){ } return [even, odd]; }",61.81759999890346,,Completions generated successfully. Ready for display in your IDE.,2206614a-946b-4b69-bb54-d0bbb6518fbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:57:26.634211,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8986,7 +8986,7 @@ class Problem { } }",197.6832000000286,,Completions generated successfully. Ready for display in your IDE.,b7659887-1e3d-40ee-8d15-79f26dd0a2e4,1,deepseek-ai/deepseek-coder-1.3b-base," if (2 * x + y < n) { -",167,1.0 +",167,1.0 2025-09-22T13:57:26.887778,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -9027,7 +9027,7 @@ def maximum(arr: List[int], k: int) -> List[int]: arr.remove(max_num)"," return sorted(get_max(arr, k)) ",253.53549999999814,,Completions generated successfully. Ready for display in your IDE.,e4a51425-fad5-4950-ac15-d3330fc2f43a,1,deepseek-ai/deepseek-coder-1.3b-base," - return [max_num] + get_max(arr, k - 1)",230,0.9884 + return [max_num] + get_max(arr, k - 1)",230,0.9884 2025-09-22T13:57:26.950736,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -9076,7 +9076,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",62.18970000190893,,Completions generated successfully. Ready for display in your IDE.,8f1c3c7b-53f6-4f6a-ad52-a291fb41225f,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:27.089311,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -9105,7 +9105,7 @@ function f(n){ return result; }",138.60169999679783,,Completions generated successfully. Ready for display in your IDE.,10671cd2-1c9d-4b77-959b-e8d1d328dc85,1,deepseek-ai/deepseek-coder-1.3b-base," factorial *= j; -",114,0.9897 +",114,0.9897 2025-09-22T13:57:27.223654,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -9145,7 +9145,7 @@ function match_parens(lst){ return 'No'; }",134.448999997403,,Completions generated successfully. Ready for display in your IDE.,d13390c8-622c-4681-b6df-031fd6444c87,1,deepseek-ai/deepseek-coder-1.3b-base," const close = ')'; -",109,1.0 +",109,1.0 2025-09-22T13:57:27.357748,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -9164,7 +9164,7 @@ function match_parens(lst){ return True ",133.54480000270996,,Completions generated successfully. Ready for display in your IDE.,0bf26430-da85-41db-bd60-fbe802b3ca06,1,deepseek-ai/deepseek-coder-1.3b-base," length = len(text) -",108,1.0 +",108,1.0 2025-09-22T13:57:27.487851,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9196,7 +9196,7 @@ class Problem { } }",130.06730000051903,,Completions generated successfully. Ready for display in your IDE.,6089f621-78d3-41bf-8aa8-7f44f582fa62,1,deepseek-ai/deepseek-coder-1.3b-base," boolean found = false; -",105,1.0 +",105,1.0 2025-09-22T13:57:27.600741,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -9238,7 +9238,7 @@ function is_sorted(lst){ }","} ",112.41700000391575,,Completions generated successfully. Ready for display in your IDE.,75c9c10a-6d25-4d57-85dc-d4c0dfa741b0,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",89,1.0 +",89,1.0 2025-09-22T13:57:27.700325,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -9270,7 +9270,7 @@ function correct_bracketing(brackets){ } }",99.86550000030547,,Completions generated successfully. Ready for display in your IDE.,84d41db9-7435-47db-889f-37e0f5e088e7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,0.9606 +",61,0.9606 2025-09-22T13:57:27.816856,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9307,7 +9307,7 @@ class Problem { }"," } }",116.15390000224579,,Completions generated successfully. Ready for display in your IDE.,3465d9ca-c9da-4b9e-9029-734c7d4c9d93,1,deepseek-ai/deepseek-coder-1.3b-base," return output; -",89,0.9922 +",89,0.9922 2025-09-22T13:57:28.270817,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -9342,7 +9342,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",453.35979999799747,,Completions generated successfully. Ready for display in your IDE.,380ba5ba-f061-413b-8930-42b87c5861c6,1,deepseek-ai/deepseek-coder-1.3b-base," if abs(numbers[i] - numbers[j]) < temp: if abs(numbers[i] - numbers[j]) < temp2: -",425,0.9872 +",425,0.9872 2025-09-22T13:57:28.328759,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -9379,7 +9379,7 @@ function is_prime(n){ return true; } ",58.02109999785898,,Completions generated successfully. Ready for display in your IDE.,33b94dbe-dd8f-48de-8b0c-906a121eb882,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:57:28.508859,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9457,7 +9457,7 @@ class Problem { } }",179.80510000052163,,Completions generated successfully. Ready for display in your IDE.,9737433a-a650-4547-b452-64cbc537302a,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 900) { -",154,1.0 +",154,1.0 2025-09-22T13:57:28.635771,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -9506,7 +9506,7 @@ function check_dict_case(dict){ return true; }",126.52830000297399,,Completions generated successfully. Ready for display in your IDE.,71713430-47d1-4a46-be1c-929d56dc9d5b,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",85,1.0 +",85,1.0 2025-09-22T13:57:29.089477,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -9554,7 +9554,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",454.04270000290126,,Completions generated successfully. Ready for display in your IDE.,ca94eccd-c7ce-47a5-85aa-7bc22a41fae6,1,deepseek-ai/deepseek-coder-1.3b-base," for i, j in [(row-1, col), (row+1, col), (row, col-1), (row, col+1)]: -",425,0.9872 +",425,0.9872 2025-09-22T13:57:29.190492,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -9580,7 +9580,7 @@ def by_length(arr: List[int]) -> List[str]: digits.reverse() for i in range(len(digits)): digits[i] = ['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight', 'Nine'][digits[i] - 1]",,100.18550000677351,,Completions generated successfully. Ready for display in your IDE.,0741bac3-0410-4f40-a6d8-d68f89c795cb,1,deepseek-ai/deepseek-coder-1.3b-base," - return digits",74,0.9866 + return digits",74,0.9866 2025-09-22T13:57:29.338034,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -9605,7 +9605,7 @@ function sum_squares(lst){ return sum; }",147.57059999828925,,Completions generated successfully. Ready for display in your IDE.,ce2ab9e2-3b60-4cf5-8dda-ec0e2ecf7b86,1,deepseek-ai/deepseek-coder-1.3b-base," var len = lst.length; -",124,1.0 +",124,1.0 2025-09-22T13:57:29.429895,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -9629,7 +9629,7 @@ function reverse_delete(s, c){ result.push(filterString, false);"," return result; }",91.21929999673739,,Completions generated successfully. Ready for display in your IDE.,84b851e1-fb11-498c-a122-b9056d4f57bc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T13:57:29.500862,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -9648,7 +9648,7 @@ function count_upper(s){ } } return result;",,71.11849999637343,,Completions generated successfully. Ready for display in your IDE.,832b13c2-12ff-44d0-bb46-40b343d03d3c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:57:29.610073,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -9690,7 +9690,7 @@ function compare_one(a, b){ } ",108.963999999105,,Completions generated successfully. Ready for display in your IDE.,9b140258-0ff5-4fbb-b336-2bdf3c362646,1,deepseek-ai/deepseek-coder-1.3b-base," return a; -",84,1.0 +",84,1.0 2025-09-22T13:57:29.705084,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9730,7 +9730,7 @@ class Problem { }); return words.get(0);",},94.53470000153175,,Completions generated successfully. Ready for display in your IDE.,d3a44206-2d0c-4b35-aaf8-2353773d2c6d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:29.821849,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -9754,7 +9754,7 @@ class Problem { ",116.54149999958463,,Completions generated successfully. Ready for display in your IDE.,31aebd60-0232-40d8-843d-3286ed9c99dc,1,deepseek-ai/deepseek-coder-1.3b-base," count -= 1 -",93,1.0 +",93,1.0 2025-09-22T13:57:29.938994,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9786,7 +9786,7 @@ class Problem { } }",117.01529999845661,,Completions generated successfully. Ready for display in your IDE.,d4bf9d84-e30d-4d30-af5e-5434c2b85e7a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",88,1.0 +",88,1.0 2025-09-22T13:57:30.137045,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -9863,7 +9863,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",198.34980000450741,,Completions generated successfully. Ready for display in your IDE.,e930c834-0a25-4891-8394-65493488c186,1,deepseek-ai/deepseek-coder-1.3b-base," if float(a_tmp) > b: -",170,1.0 +",170,1.0 2025-09-22T13:57:30.289828,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -9907,7 +9907,7 @@ function bf(planet1, planet2){ return res; }",152.6942999989842,,Completions generated successfully. Ready for display in your IDE.,090042a7-ada8-4cc4-a646-dceb1a557202,1,deepseek-ai/deepseek-coder-1.3b-base," let temp = position1; -",120,0.9956 +",120,0.9956 2025-09-22T13:57:30.432780,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9985,7 +9985,7 @@ class Problem { } }",142.35139999800595,,Completions generated successfully. Ready for display in your IDE.,e58e038b-15d9-4c70-a571-8edf0ab8bb3b,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""xc""; -",113,1.0 +",113,1.0 2025-09-22T13:57:31.137703,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -10013,7 +10013,7 @@ function parse_music(music_string){ if (isNaN(Number(s))) { switch (s) { case ""o"": - song.push",668,0.9548 + song.push",668,0.9548 2025-09-22T13:57:31.251251,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -10035,7 +10035,7 @@ function fizz_buzz(n){ }","} ",113.78700000204844,,Completions generated successfully. Ready for display in your IDE.,dea52330-2256-40ce-8257-4b660544515b,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",90,1.0 +",90,1.0 2025-09-22T13:57:31.478134,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10079,7 +10079,7 @@ class Problem { } }",226.05060000205413,,Completions generated successfully. Ready for display in your IDE.,19fa7520-baf8-476d-b4d9-f750417ec6b6,1,deepseek-ai/deepseek-coder-1.3b-base," else if (op.get(i).equals(""*"")) -",199,1.0 +",199,1.0 2025-09-22T13:57:31.839934,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -10097,7 +10097,7 @@ function rolling_max(numbers){ tempMax = Math.max(tempMax, numbers[i]); } max.push(tempMax); -",339,0.9344 +",339,0.9344 2025-09-22T13:57:31.923867,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -10125,7 +10125,7 @@ function triples_sum_to_zero(l){ } ",84.09209999808809,,Completions generated successfully. Ready for display in your IDE.,9592b9a7-3978-48d7-bef4-fc0389f3001d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:57:32.011647,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10154,7 +10154,7 @@ class Problem { if (n == 3) return 0; return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4);",},87.48880000348436,,Completions generated successfully. Ready for display in your IDE.,0cf7a87d-92af-465d-99d2-feb2fdfdaf4a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:32.055776,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -10179,7 +10179,7 @@ class Problem { else: return False -",44.04920000524726,,Completions generated successfully. Ready for display in your IDE.,ae0884cb-e92a-48b8-9ae4-cb1bc6efcd52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",44.04920000524726,,Completions generated successfully. Ready for display in your IDE.,ae0884cb-e92a-48b8-9ae4-cb1bc6efcd52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:32.195178,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10230,7 +10230,7 @@ class Problem { return ""Yes""; } }",138.54100000025937,,Completions generated successfully. Ready for display in your IDE.,b3882d2b-afa2-40fb-8230-b01e32914b7c,1,deepseek-ai/deepseek-coder-1.3b-base," String result = ""No""; -",111,1.0 +",111,1.0 2025-09-22T13:57:32.344436,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10283,7 +10283,7 @@ class Problem { newArr.add((long)minIndex);"," } }",149.72470000066096,,Completions generated successfully. Ready for display in your IDE.,6b508cf0-f53e-4808-9455-7fd32ff43c29,1,deepseek-ai/deepseek-coder-1.3b-base," return newArr; -",121,1.0 +",121,1.0 2025-09-22T13:57:32.543683,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10322,7 +10322,7 @@ class Problem { return res; } }",198.70369999989634,,Completions generated successfully. Ready for display in your IDE.,2dbee4fc-af82-418c-8747-1d7b60ff3619,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList res = new ArrayList(); -",174,1.0 +",174,1.0 2025-09-22T13:57:32.638333,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -10361,7 +10361,7 @@ function find_max(words){ return max_unique_word; }",94.81599999708124,,Completions generated successfully. Ready for display in your IDE.,14f68593-099d-453b-9088-00842445ff61,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:32.818044,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -10392,7 +10392,7 @@ function total_match(lst1, lst2){ } ",179.1808999987552,,Completions generated successfully. Ready for display in your IDE.,9a8b1fc7-1c40-44e6-87b7-c6f420a27281,1,deepseek-ai/deepseek-coder-1.3b-base," let sum_lst2 = 0; -",150,1.0 +",150,1.0 2025-09-22T13:57:32.924797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10441,7 +10441,7 @@ class Problem { } }",106.6011000002618,,Completions generated successfully. Ready for display in your IDE.,ccca4d7f-c8fd-4428-a882-f4bfda5b64e9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:57:33.047103,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -10518,7 +10518,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",121.773500002746,,Completions generated successfully. Ready for display in your IDE.,d9bb14db-cb34-42f4-a427-d7ec48c4587a,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:57:33.113765,[1],"//Return array with elements incremented by 1. // >>> incr_list([1, 2, 3]) // [2, 3, 4] @@ -10530,7 +10530,7 @@ function incr_list(l){ new_list.push(l[i] + 1); } return new_list;",,67.1537999951397,,Completions generated successfully. Ready for display in your IDE.,a491cf90-cbad-4466-bf97-f511edb746e0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",46,1.0 +}",46,1.0 2025-09-22T13:57:33.208797,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -10579,7 +10579,7 @@ function check_dict_case(dict){ return true; }",94.46890000253916,,Completions generated successfully. Ready for display in your IDE.,135ab0f0-efe0-4006-90ad-92eded65f1a4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:57:33.902810,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -10609,7 +10609,7 @@ function vowels_count(s){ } console.log('VOWELS COUNT'); console.log (vowels_count ('abcde')); //should return 2 -console.",667,0.9079 +console.",667,0.9079 2025-09-22T13:57:33.998191,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -10634,7 +10634,7 @@ function sum_squares(lst){ } }",},94.67959999892628,,Completions generated successfully. Ready for display in your IDE.,1127e649-339d-4209-ab78-006e366a97ad,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",72,1.0 +",72,1.0 2025-09-22T13:57:34.704708,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -10681,7 +10681,7 @@ def pluck(arr: List[int]) -> List[int]: for i, e in enumerate(evens): if e == min_even: if min_index is not None: - if arr[min_index] > arr[i]:",679,0.9647 + if arr[min_index] > arr[i]:",679,0.9647 2025-09-22T13:57:35.102508,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -10727,7 +10727,7 @@ function histogram(test){ //console.log('max', max) for (var i=0 ; i < lst.length ; ++i ) { -",372,0.9213 +",372,0.9213 2025-09-22T13:57:35.220389,[1],"//xs represent coefficients of a polynomial. // xs[0] + xs[1] * x + xs[2] * x^2 + .... // Return derivative of this polynomial in the same form. @@ -10742,7 +10742,7 @@ function derivative(xs){ }","} ",116.87019999953918,,Completions generated successfully. Ready for display in your IDE.,18089e30-5c24-4a0d-b9b2-d621eac6f776,1,deepseek-ai/deepseek-coder-1.3b-base," return coeff; -",93,0.9915 +",93,0.9915 2025-09-22T13:57:35.348493,[1],"from typing import List def strange_sort_list(lst: List[int]) -> List[int]: @@ -10766,7 +10766,7 @@ def strange_sort_list(lst: List[int]) -> List[int]: out.append(lst.pop()) return out",128.48939999821596,,Completions generated successfully. Ready for display in your IDE.,438a2518-636d-4a04-86be-83bd875f3620,1,deepseek-ai/deepseek-coder-1.3b-base," if not lst: -",104,1.0 +",104,1.0 2025-09-22T13:57:35.463439,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10803,7 +10803,7 @@ class Problem { } }",114.80079999455484,,Completions generated successfully. Ready for display in your IDE.,4b9dace2-7492-4803-b8dd-61e5f363d7c1,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",91,1.0 +",91,1.0 2025-09-22T13:57:35.735598,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10825,7 +10825,7 @@ class Problem { BigInteger nSquared = nBigInteger.multiply(nBigInteger);"," } }",271.9084999989718,,Completions generated successfully. Ready for display in your IDE.,35292fcf-1c96-4abc-b494-a77d267ffb9f,1,deepseek-ai/deepseek-coder-1.3b-base," return nSquared.divide(new BigInteger(""2"")).longValue(); -",248,1.0 +",248,1.0 2025-09-22T13:57:35.986174,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -10897,7 +10897,7 @@ function minPath(grid, k){ return path; }",250.50129999726778,,Completions generated successfully. Ready for display in your IDE.,182adbc7-1666-44bc-956b-f61337f169bc,1,deepseek-ai/deepseek-coder-1.3b-base," if(!path || !startCell && curr >= startCell){ -",219,0.9666 +",219,0.9666 2025-09-22T13:57:36.072649,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10917,7 +10917,7 @@ class Problem { } return 1;",},85.35300000221469,,Completions generated successfully. Ready for display in your IDE.,96560824-a38c-4b63-a5cc-2a023f3b8870,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:36.115688,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -10939,7 +10939,7 @@ class Problem { if string[i:] == prefix[:len(string) - i]: return string + prefix[len(string) - i:] "," -",42.84209999605082,,Completions generated successfully. Ready for display in your IDE.,285c2546-3cca-45da-ad1d-e7558bd6b2c5,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",42.84209999605082,,Completions generated successfully. Ready for display in your IDE.,285c2546-3cca-45da-ad1d-e7558bd6b2c5,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:36.221367,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -10982,7 +10982,7 @@ def pluck(arr: List[int]) -> List[int]: return [min(evens), arr.index(min(evens))] ",105.39699999935692,,Completions generated successfully. Ready for display in your IDE.,b8bc0f63-99ee-4c52-9aef-13c79b6dbe74,1,deepseek-ai/deepseek-coder-1.3b-base," return [] -",78,1.0 +",78,1.0 2025-09-22T13:57:36.309459,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11011,7 +11011,7 @@ class Problem { } }",87.98830000159796,,Completions generated successfully. Ready for display in your IDE.,0bc41766-52c5-43aa-8486-b70895b1012e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,0.9493 +",62,0.9493 2025-09-22T13:57:36.569523,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11042,7 +11042,7 @@ class Problem { } }",259.99310000042897,,Completions generated successfully. Ready for display in your IDE.,b2ae10b5-ebdd-427f-bc75-d6de37cece3e,1,deepseek-ai/deepseek-coder-1.3b-base," int n_num = Integer.parseInt(n_nums[0]); -",234,1.0 +",234,1.0 2025-09-22T13:57:36.662023,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11077,7 +11077,7 @@ class Problem { } }",92.47750000213273,,Completions generated successfully. Ready for display in your IDE.,db4b401b-85c3-4fd9-bbb5-3a13e64e15ce,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:36.819909,[1],"def x_or_y(n: int, x: int, y: int) -> int: """"""A simple program which should return the value of x if n is a prime number and should return the value of y otherwise. @@ -11096,7 +11096,7 @@ class Problem { return x ",157.34779999911552,,Completions generated successfully. Ready for display in your IDE.,63b82da1-64cc-422c-96af-ba5bb6c0014c,1,deepseek-ai/deepseek-coder-1.3b-base," if n == 1 or n == 2: -",137,0.9841 +",137,0.9841 2025-09-22T13:57:36.968140,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -11168,7 +11168,7 @@ function minPath(grid, k){ return path; }",147.83929999975953,,Completions generated successfully. Ready for display in your IDE.,98f57472-893c-4ab7-9a71-635b2523e54a,1,deepseek-ai/deepseek-coder-1.3b-base," x = i; -",114,1.0 +",114,1.0 2025-09-22T13:57:37.075328,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11191,7 +11191,7 @@ class Problem { } }",106.95600000326522,,Completions generated successfully. Ready for display in your IDE.,2ce3a5f2-6b04-4004-a115-8e4662e418e1,1,deepseek-ai/deepseek-coder-1.3b-base," return a; -",84,1.0 +",84,1.0 2025-09-22T13:57:37.304417,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -11216,7 +11216,7 @@ class Problem { ",229.196200001752,,Completions generated successfully. Ready for display in your IDE.,3d4d0259-a049-4b81-9996-8a8a4a9656be,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, int(num / 2)): -",206,0.9503 +",206,0.9503 2025-09-22T13:57:37.358933,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -11245,7 +11245,7 @@ function parse_nested_parens(paren_string){ } ",54.260299999441486,,Completions generated successfully. Ready for display in your IDE.,7e475540-f63e-4c3f-b4f5-b5827bab2eb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:57:37.575797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11283,7 +11283,7 @@ class Problem { } }",216.45530000387225,,Completions generated successfully. Ready for display in your IDE.,6ca3044e-1a91-4113-a6c5-3d18f179a210,1,deepseek-ai/deepseek-coder-1.3b-base," char[] chars = music_string.toCharArray(); -",192,1.0 +",192,1.0 2025-09-22T13:57:37.967111,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11318,7 +11318,7 @@ class Problem { } }",391.36149999831105,,Completions generated successfully. Ready for display in your IDE.,b27a95f8-597c-45ee-98f0-50439c0b120f,1,deepseek-ai/deepseek-coder-1.3b-base," lst.set(i, lst.get(i) * lst.get(i) * lst.get(i)); -",366,1.0 +",366,1.0 2025-09-22T13:57:38.105971,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -11342,7 +11342,7 @@ class Problem { ",138.46879999618977,,Completions generated successfully. Ready for display in your IDE.,2539fee9-005c-43a8-ab09-6f4eae9fa912,1,deepseek-ai/deepseek-coder-1.3b-base," if char == '(': -",116,1.0 +",116,1.0 2025-09-22T13:57:38.203426,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -11357,7 +11357,7 @@ function filter_integers(values){ });","} ",97.34020000178134,,Completions generated successfully. Ready for display in your IDE.,f7a367ca-cdc4-454c-ba29-add6e861fcfe,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",73,1.0 +",73,1.0 2025-09-22T13:57:38.349617,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11393,7 +11393,7 @@ class Problem { } }",146.31880000524689,,Completions generated successfully. Ready for display in your IDE.,f440d873-cfac-4715-95cc-5f3dfb0c8c34,1,deepseek-ai/deepseek-coder-1.3b-base," long max = 0L; -",120,0.8516 +",120,0.8516 2025-09-22T13:57:38.415179,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -11413,7 +11413,7 @@ function double_the_difference(lst){ return sum; } ",64.90409999969415,,Completions generated successfully. Ready for display in your IDE.,ba678e97-69bd-4b91-ae62-55136634bd08,1,deepseek-ai/deepseek-coder-1.3b-base," - ",41,1.0 + ",41,1.0 2025-09-22T13:57:38.504405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11456,7 +11456,7 @@ class Problem { } }",88.79719999822555,,Completions generated successfully. Ready for display in your IDE.,fa33165e-5391-45ae-ac78-3492196f9bdf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,0.9169 +",64,0.9169 2025-09-22T13:57:38.622865,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11482,7 +11482,7 @@ class Problem { } }",118.54229999880772,,Completions generated successfully. Ready for display in your IDE.,c7a76f67-c2fc-4fbb-aa53-707c0349c252,1,deepseek-ai/deepseek-coder-1.3b-base," chosen = i; -",93,1.0 +",93,1.0 2025-09-22T13:57:38.916038,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -11509,7 +11509,7 @@ function separate_paren_groups(paren_string){ ",292.9230000008829,,Completions generated successfully. Ready for display in your IDE.,40a8f66f-8b0d-4592-bd92-6346d029c015,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0; i List[int]: @@ -11555,7 +11555,7 @@ def get_positive(l: List[int]) -> List[int]: return p ",123.42699999862816,,Completions generated successfully. Ready for display in your IDE.,21b05f12-82a2-4580-8564-32ecc3d00a4e,1,deepseek-ai/deepseek-coder-1.3b-base," for i in l: -",102,1.0 +",102,1.0 2025-09-22T13:57:39.355640,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -11574,7 +11574,7 @@ function sum_product(numbers){ return [sum, product]; } ",229.40229999949224,,Completions generated successfully. Ready for display in your IDE.,044eebe4-2f90-41f0-913b-07ee07445aa7,1,deepseek-ai/deepseek-coder-1.3b-base," if (numbers === [] || !Array.isArray(numbers)){ -",206,0.958 +",206,0.958 2025-09-22T13:57:39.504649,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -11618,7 +11618,7 @@ function bf(planet1, planet2){ return res; }",148.7769999948796,,Completions generated successfully. Ready for display in your IDE.,b934bca5-3112-4599-a058-c2c1371b5dbd,1,deepseek-ai/deepseek-coder-1.3b-base," 'Neptune':8 -",123,1.0 +",123,1.0 2025-09-22T13:57:39.738888,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -11659,7 +11659,7 @@ function match_parens(lst){ }",234.08780000318075,,Completions generated successfully. Ready for display in your IDE.,9f13c3f0-6155-4de6-a1ac-59060402b327,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here... const str1 = lst[0]; -",198,0.9844 +",198,0.9844 2025-09-22T13:57:39.803731,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -11675,7 +11675,7 @@ function how_many_times(string, substring){ i = string.indexOf(substring, i) + 1; } return count;",,65.19850000040606,,Completions generated successfully. Ready for display in your IDE.,7a37e4c4-f021-4ade-a87e-63db06a3702b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:57:39.891810,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11703,7 +11703,7 @@ class Problem { public static long digitSum(String s) { return s.chars().filter(i -> i >= 65 && i <= 90).sum();",},87.61650000087684,,Completions generated successfully. Ready for display in your IDE.,185cd0ae-ae57-46fc-b73d-a4796fc84998,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:40.045044,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -11725,7 +11725,7 @@ function find_closest_elements(numbers){ } ",152.44019999954617,,Completions generated successfully. Ready for display in your IDE.,0f1485ee-d806-4952-8f8a-3a4b5f753f02,1,deepseek-ai/deepseek-coder-1.3b-base," let b = numbers[1]; -",124,1.0 +",124,1.0 2025-09-22T13:57:40.293777,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -11798,7 +11798,7 @@ function minPath(grid, k){ }",248.63899999763817,,Completions generated successfully. Ready for display in your IDE.,d333614a-f12e-4d78-967c-d6084c31e476,1,deepseek-ai/deepseek-coder-1.3b-base," //find starting point let startCell = Infinity; -",218,0.9958 +",218,0.9958 2025-09-22T13:57:40.706279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11838,7 +11838,7 @@ class Problem { } }",412.166400004935,,Completions generated successfully. Ready for display in your IDE.,0d052b62-2e99-4410-aeaa-b246f6738405,1,deepseek-ai/deepseek-coder-1.3b-base," memo[i] = memo[i - 1] + memo[i - 2] + memo[i - 3]; -",387,1.0 +",387,1.0 2025-09-22T13:57:40.833570,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -11868,7 +11868,7 @@ function words_in_sentence(sentence){ }"," }).join(' '); }",127.12800000008428,,Completions generated successfully. Ready for display in your IDE.,88b865be-e8c4-4ab1-9628-7f043494ea05,1,deepseek-ai/deepseek-coder-1.3b-base," return isPrime; -",103,1.0 +",103,1.0 2025-09-22T13:57:40.927936,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11900,7 +11900,7 @@ class Problem { } return result;",},93.97840000019642,,Completions generated successfully. Ready for display in your IDE.,40f20329-a8fb-472c-a6ef-42f0100e6f9a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:41.033591,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11932,7 +11932,7 @@ class Problem { } }",105.72959999990417,,Completions generated successfully. Ready for display in your IDE.,a6406e38-f822-478b-9db8-5fb47ed2daea,1,deepseek-ai/deepseek-coder-1.3b-base," continue; -",79,0.9629 +",79,0.9629 2025-09-22T13:57:41.239472,[1],"from typing import List, Any def filter_integers(values: List[Any]) -> List[int]: @@ -11942,7 +11942,7 @@ def filter_integers(values: List[Any]) -> List[int]: >>> filter_integers([1, 2, 3, 'abc', { }, []]) [1, 2, 3] """""" -",,205.68839999759803,,Completions generated successfully. Ready for display in your IDE.,f7b3f392-fffe-4d66-b3d8-dc8f2c7b1160,1,deepseek-ai/deepseek-coder-1.3b-base," return [value for value in values if isinstance(value, int)]",184,0.9515 +",,205.68839999759803,,Completions generated successfully. Ready for display in your IDE.,f7b3f392-fffe-4d66-b3d8-dc8f2c7b1160,1,deepseek-ai/deepseek-coder-1.3b-base," return [value for value in values if isinstance(value, int)]",184,0.9515 2025-09-22T13:57:41.398157,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -11961,7 +11961,7 @@ def sort_even(l: List[int]) -> List[int]: l[i] = temp[i // 2] return l",158.19879999617115,,Completions generated successfully. Ready for display in your IDE.,ba19b971-4494-4f27-8621-af507027f5d9,1,deepseek-ai/deepseek-coder-1.3b-base," temp.append(l[i]) -",134,0.9946 +",134,0.9946 2025-09-22T13:57:41.459705,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -12004,7 +12004,7 @@ function histogram(test){ return result; } ",61.33199999749195,,Completions generated successfully. Ready for display in your IDE.,f668da9f-f892-4939-b8ad-04cf44a5e8c4,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:41.697691,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12026,7 +12026,7 @@ class Problem { } }",238.3031000063056,,Completions generated successfully. Ready for display in your IDE.,8cedf7cb-5d25-40d8-8de3-f74ed47b272f,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = lower; i <= upper; i++) { -",214,1.0 +",214,1.0 2025-09-22T13:57:41.795569,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -12048,7 +12048,7 @@ function can_arrange(arr){ } ",97.18820000125561,,Completions generated successfully. Ready for display in your IDE.,f07ef9db-46c4-43f5-8a36-6d96c2fd6610,1,deepseek-ai/deepseek-coder-1.3b-base," continue; -",74,1.0 +",74,1.0 2025-09-22T13:57:42.225916,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -12075,7 +12075,7 @@ def sort_array(array: List[int]) -> List[int]: array.sort() else: array.sort(reverse=True) -",403,0.9657 +",403,0.9657 2025-09-22T13:57:42.416951,[1],"//For a given array of input numbers, calculate Mean Absolute Deviation // around the mean of this dataset. // Mean Absolute Deviation is the average absolute difference between each @@ -12095,7 +12095,7 @@ function mean_absolute_deviation(numbers){ }","} ",191.20030000340194,,Completions generated successfully. Ready for display in your IDE.,5c470597-966d-4d9c-999e-5eab81c2570d,1,deepseek-ai/deepseek-coder-1.3b-base," return sum_abs_diff / numbers.length; -",166,0.9809 +",166,0.9809 2025-09-22T13:57:42.871879,[1],"from typing import List def minSubArraySum(nums: List[int]) -> int: @@ -12112,7 +12112,7 @@ def minSubArraySum(nums: List[int]) -> int: ",454.7935000009602,,Completions generated successfully. Ready for display in your IDE.,dc827082-3b6f-49d7-9924-784a861699cb,1,deepseek-ai/deepseek-coder-1.3b-base," if nums[i] < nums[i - 1]: nums[i] = nums[i - 1] + nums[i] -",431,0.9923 +",431,0.9923 2025-09-22T13:57:43.042832,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -12167,7 +12167,7 @@ function by_length(arr){ return arr3; }",171.08600000210572,,Completions generated successfully. Ready for display in your IDE.,6bbd00d7-9948-4991-a560-72e39c8a9ebe,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Eight""); -",144,1.0 +",144,1.0 2025-09-22T13:57:43.121738,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -12201,7 +12201,7 @@ function count_nums(arr){ }"," return count; }",78.18570000381442,,Completions generated successfully. Ready for display in your IDE.,37c2b707-51d8-4340-b1d9-12f542173105,1,deepseek-ai/deepseek-coder-1.3b-base," } -",54,1.0 +",54,1.0 2025-09-22T13:57:43.228634,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -12268,7 +12268,7 @@ function numerical_letter_grade(grades){ return grades_array; }",106.74990000552498,,Completions generated successfully. Ready for display in your IDE.,9418768c-11c9-49b4-8eda-ed6ecde0cbd2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",78,1.0 +",78,1.0 2025-09-22T13:57:43.318505,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12293,7 +12293,7 @@ class Problem { public static boolean sameChars(String s0, String s1) { return s0.chars().distinct().sorted().boxed().collect(Collectors.toList()).equals(s1.chars().distinct().sorted().boxed().collect(Collectors.toList()));",},89.36040000116918,,Completions generated successfully. Ready for display in your IDE.,76128692-364e-4308-905a-4702efc24dde,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:43.405146,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -12331,7 +12331,7 @@ function is_prime(n){ } ",86.8553000036627,,Completions generated successfully. Ready for display in your IDE.,dcfe468e-7bf8-4029-b617-79239e9575d2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:43.572973,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12371,7 +12371,7 @@ class Problem { } }",166.85599999618717,,Completions generated successfully. Ready for display in your IDE.,5cc1e1c5-381a-4953-aa17-ace3d7e93ae0,1,deepseek-ai/deepseek-coder-1.3b-base," for (String word: words) { -",140,0.9872 +",140,0.9872 2025-09-22T13:57:43.828199,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -12385,7 +12385,7 @@ function filter_by_substring(strings, substring){ return filtered; }",255.46819999726722,,Completions generated successfully. Ready for display in your IDE.,db52aba6-f8af-4049-a7ab-2679a5114b1e,1,deepseek-ai/deepseek-coder-1.3b-base," if (strings[i].indexOf(substring) !== -1 ){ -",233,0.9656 +",233,0.9656 2025-09-22T13:57:43.939621,[1],"//Given the lengths of the three sides of a triangle. Return true if the three // sides form a right-angled triangle, false otherwise. // A right-angled triangle is a triangle in which one angle is right angle or @@ -12402,7 +12402,7 @@ function right_angle_triangle(a, b, c){ } ",110.98589999892283,,Completions generated successfully. Ready for display in your IDE.,92762183-1431-4a5f-9e77-d1e444f789ba,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",84,1.0 +",84,1.0 2025-09-22T13:57:44.039953,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12443,7 +12443,7 @@ class Problem { } return res;",},100.25350000069011,,Completions generated successfully. Ready for display in your IDE.,07d354ab-84a9-4f13-9cf5-c360eb05f42d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T13:57:44.262025,[1],"//Given an array of non-negative integers, return a cojs of the given array after sorting, // you will sort the given array in ascending order if the sum( first index value, last index value) is odd, // or sort it in descending order if the sum( first index value, last index value) is even. @@ -12468,7 +12468,7 @@ function sort_array(array){ }",221.56649999669753,,Completions generated successfully. Ready for display in your IDE.,04949976-02d3-4cb8-92a3-fe27c4b2ccea,1,deepseek-ai/deepseek-coder-1.3b-base," //if (sum % 2 === 0 ) { - ",197,0.8935 + ",197,0.8935 2025-09-22T13:57:44.408726,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -12512,7 +12512,7 @@ function get_odd_collatz(n){ ",146.12109999870881,,Completions generated successfully. Ready for display in your IDE.,1ed34b96-f82f-4ef4-af02-cc27174aea8b,1,deepseek-ai/deepseek-coder-1.3b-base," while (true){ -",121,0.9873 +",121,0.9873 2025-09-22T13:57:44.563800,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12549,7 +12549,7 @@ class Problem { } }",155.78930000629043,,Completions generated successfully. Ready for display in your IDE.,c991e16d-9680-4569-997d-37c7d8f4d218,1,deepseek-ai/deepseek-coder-1.3b-base," temp.append(c); -",132,1.0 +",132,1.0 2025-09-22T13:57:44.702757,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -12574,7 +12574,7 @@ function sort_numbers(numbers){ return numString; }",137.92580000153976,,Completions generated successfully. Ready for display in your IDE.,78fb89e8-5a6a-4927-803f-808a75caf906,1,deepseek-ai/deepseek-coder-1.3b-base," ""eight"": 8, -",115,1.0 +",115,1.0 2025-09-22T13:57:44.769114,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12597,7 +12597,7 @@ class Problem { return result; } }",66.68109999736771,,Completions generated successfully. Ready for display in your IDE.,22575594-d402-4f13-8ae7-e926ef346c47,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,0.9407 + ",42,0.9407 2025-09-22T13:57:44.876949,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -12640,7 +12640,7 @@ function get_odd_collatz(n){ } ",107.313400003477,,Completions generated successfully. Ready for display in your IDE.,5b857e86-6b09-4380-be3c-793094cb3336,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",82,1.0 +",82,1.0 2025-09-22T13:57:44.956725,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -12667,7 +12667,7 @@ function next_smallest(lst){ return second_smallest; }",79.17820000147913,,Completions generated successfully. Ready for display in your IDE.,34da66e3-3d1e-45c2-b2ed-299bdcaa78f0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:57:45.303707,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -12736,7 +12736,7 @@ function do_algebra(operator, operand){ ",346.9537000055425,,Completions generated successfully. Ready for display in your IDE.,a4fc3169-428e-4b26-b959-c9b30831ae71,1,deepseek-ai/deepseek-coder-1.3b-base," if(!do_again) {return answer;} else{return do_algebra(operator, operand)}; -",320,0.9883 +",320,0.9883 2025-09-22T13:57:45.411959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12769,7 +12769,7 @@ class Problem { } }",108.53489999863086,,Completions generated successfully. Ready for display in your IDE.,3acaff7f-e14c-45ba-b496-2c802d47a4c3,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",84,1.0 +",84,1.0 2025-09-22T13:57:45.475644,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12808,7 +12808,7 @@ class Problem { }"," return b; } }",63.00409999676049,,Completions generated successfully. Ready for display in your IDE.,2c51da2b-298e-4837-82e1-d521e9c83dcc,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:45.654760,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -12847,7 +12847,7 @@ function valid_date(date){ } ",179.32830000063404,,Completions generated successfully. Ready for display in your IDE.,faec5c26-c06a-4d34-9794-d69389524834,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9169 +",69,0.9169 2025-09-22T13:57:45.730288,[1],"//Write a function that takes an integer a and returns true // if this ingeger is a cube of some integer number. // Note: you may assume the input is always valid. @@ -12870,7 +12870,7 @@ function iscube(a){ }else{ return false;",},74.7655000013765,,Completions generated successfully. Ready for display in your IDE.,eeeca37f-087d-4d4f-8936-0bec8facb406,1,deepseek-ai/deepseek-coder-1.3b-base," } -",52,1.0 +",52,1.0 2025-09-22T13:57:46.426874,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -12893,7 +12893,7 @@ function cycpattern_check(a, b){ ",696.9198000006145,,Completions generated successfully. Ready for display in your IDE.,70991dda-ac5c-487d-8891-380077cc79d3,1,deepseek-ai/deepseek-coder-1.3b-base," //console.log('b: '+i); console.log((new RegExp(`^${b}(?=${a})|\\Q${b}\\E(\\Q${b}\\E|${a}.*?)+$`)).test(a)); -",669,0.8914 +",669,0.8914 2025-09-22T13:57:46.678118,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12935,7 +12935,7 @@ class Problem { } }",250.5167999988771,,Completions generated successfully. Ready for display in your IDE.,4310919a-47d6-4ae8-8d52-d9dbf8fc2780,1,deepseek-ai/deepseek-coder-1.3b-base," if (! Character.isLetter(s.charAt(i))){ -",223,0.9489 +",223,0.9489 2025-09-22T13:57:46.720212,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -12962,7 +12962,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: if l[i] + l[j] + l[k] == 0: return True return False -",41.92530000000261,,Completions generated successfully. Ready for display in your IDE.,76c0b9cc-eb9f-4992-b32c-92dba1d83bb0,1,deepseek-ai/deepseek-coder-1.3b-base,,19,1.0 +",41.92530000000261,,Completions generated successfully. Ready for display in your IDE.,76c0b9cc-eb9f-4992-b32c-92dba1d83bb0,1,deepseek-ai/deepseek-coder-1.3b-base,,19,1.0 2025-09-22T13:57:46.824249,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -12973,7 +12973,7 @@ function all_prefixes(string){ }","} ",103.99909999978263,,Completions generated successfully. Ready for display in your IDE.,56905171-f84b-44c8-bbb0-f21c2586db3c,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",82,1.0 +",82,1.0 2025-09-22T13:57:46.918190,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -13002,7 +13002,7 @@ function f(n){ return result; }",94.01859999343287,,Completions generated successfully. Ready for display in your IDE.,273f4e8c-0494-438f-8ae3-94834c93e5e7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:47.026056,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -13027,7 +13027,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",107.04709999845363,,Completions generated successfully. Ready for display in your IDE.,3b28ebb0-0339-4e9c-a0cb-7739f6176e8e,1,deepseek-ai/deepseek-coder-1.3b-base," group = '' -",83,1.0 +",83,1.0 2025-09-22T13:57:47.211475,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13064,7 +13064,7 @@ class Problem { } }",185.25640000007115,,Completions generated successfully. Ready for display in your IDE.,d6fe6a86-52d5-4d7e-a582-d9698b2fe85b,1,deepseek-ai/deepseek-coder-1.3b-base," if (open_paren == 0) { -",163,1.0 +",163,1.0 2025-09-22T13:57:47.306241,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -13101,7 +13101,7 @@ function move_one_ball(arr){ } }",94.72920000553131,,Completions generated successfully. Ready for display in your IDE.,40a06913-6af0-46d8-84d1-af9ba6ff90b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:47.569502,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -13124,7 +13124,7 @@ function odd_count(lst){ return new_lst; }",263.01059999968857,,Completions generated successfully. Ready for display in your IDE.,4ab11bd1-125a-48c6-be2e-266ae6328252,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < lst.length; i++) { -",238,0.9962 +",238,0.9962 2025-09-22T13:57:47.798056,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13164,7 +13164,7 @@ class Problem { } }",228.28669999580598,,Completions generated successfully. Ready for display in your IDE.,afe7df12-c7e3-4c23-a1b0-859920342f87,1,deepseek-ai/deepseek-coder-1.3b-base," if (isPrime == true && i != 1) { -",203,0.9587 +",203,0.9587 2025-09-22T13:57:48.022326,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -13220,7 +13220,7 @@ function by_length(arr){ return arr3; }",224.68769999977667,,Completions generated successfully. Ready for display in your IDE.,eb7a4383-8afb-4007-b420-8a00eaf30699,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... var arr2 = new Array(); -",193,0.929 +",193,0.929 2025-09-22T13:57:48.128292,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -13287,7 +13287,7 @@ function numerical_letter_grade(grades){ return grades_array; }",105.73129999829689,,Completions generated successfully. Ready for display in your IDE.,0ed43ab1-8ed8-4a86-b252-b890cf21207a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:57:48.236162,[1],"from typing import List def add_elements(arr: List[int], k: int) -> int: @@ -13309,7 +13309,7 @@ def add_elements(arr: List[int], k: int) -> int: if num < 100:"," return total ",107.33970000001136,,Completions generated successfully. Ready for display in your IDE.,3717eab6-2be5-4e98-a124-3e0afbd4c3cb,1,deepseek-ai/deepseek-coder-1.3b-base," total += num -",84,1.0 +",84,1.0 2025-09-22T13:57:48.414096,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -13332,7 +13332,7 @@ function double_the_difference(lst){ continue; } } -",155,1.0 +",155,1.0 2025-09-22T13:57:48.567959,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -13354,7 +13354,7 @@ function anti_shuffle(s){ return result.join("" ""); }",153.84079999785172,,Completions generated successfully. Ready for display in your IDE.,767112fd-1aab-43f1-afaa-44ba70aabed2,1,deepseek-ai/deepseek-coder-1.3b-base," strings.forEach((str) => { -",131,0.9949 +",131,0.9949 2025-09-22T13:57:48.682111,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13392,7 +13392,7 @@ class Problem { } }",114.51379999925848,,Completions generated successfully. Ready for display in your IDE.,69ede347-2f7c-4679-afff-059865201795,1,deepseek-ai/deepseek-coder-1.3b-base," min++; -",89,1.0 +",89,1.0 2025-09-22T13:57:48.967247,[1],"from typing import List def rescale_to_unit(numbers: List[float]) -> List[float]: @@ -13405,7 +13405,7 @@ def rescale_to_unit(numbers: List[float]) -> List[float]: if _min == _max: return [0.5] * len(numbers)",,284.91809999832185,,Completions generated successfully. Ready for display in your IDE.,e107b80f-90b4-40dc-ab6c-ec165c0db2df,1,deepseek-ai/deepseek-coder-1.3b-base," else: - return [_value / (_max - _min) for _value in numbers]",262,0.9557 + return [_value / (_max - _min) for _value in numbers]",262,0.9557 2025-09-22T13:57:49.010723,[1],"//Insert a number 'delimeter' between every two consecutive elements of input array `numbers' // >>> intersperse([], 4) // [] @@ -13419,7 +13419,7 @@ function intersperse(numbers, delimeter){ result.push(delimeter); } } - return result;",,42.96790000080364,,Completions generated successfully. Ready for display in your IDE.,91a71f47-2650-4dd5-aa56-817cc866490c,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 + return result;",,42.96790000080364,,Completions generated successfully. Ready for display in your IDE.,91a71f47-2650-4dd5-aa56-817cc866490c,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:49.136894,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13459,7 +13459,7 @@ class Problem { }"," } }",126.19529999938095,,Completions generated successfully. Ready for display in your IDE.,e5b447fc-f4e1-42ce-b657-e765f394b868,1,deepseek-ai/deepseek-coder-1.3b-base," return arrayList; -",100,1.0 +",100,1.0 2025-09-22T13:57:49.437945,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13489,7 +13489,7 @@ class Problem { } }",300.56770000373945,,Completions generated successfully. Ready for display in your IDE.,0226d0cb-02f7-43ad-9d24-c667f7e69b80,1,deepseek-ai/deepseek-coder-1.3b-base," if (!vowels.contains(text.substring(i, i + 1))) { -",278,0.9814 +",278,0.9814 2025-09-22T13:57:49.534973,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -13507,7 +13507,7 @@ function choose_num(x, y){ }","} ",96.73479999764822,,Completions generated successfully. Ready for display in your IDE.,bcad8828-8500-44d8-b39a-e81810c1dbbb,1,deepseek-ai/deepseek-coder-1.3b-base," return num; -",72,1.0 +",72,1.0 2025-09-22T13:57:49.623862,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -13529,7 +13529,7 @@ function choose_num(x, y){ if (s[i] == 'y' and i == len(s) - 1): count += 1 i += 1",,88.99490000476362,,Completions generated successfully. Ready for display in your IDE.,695d1cf4-8f49-4b47-b9a2-20f76d02c1df,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",63,1.0 + return count",63,1.0 2025-09-22T13:57:49.886484,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -13553,7 +13553,7 @@ function vowels_count(s){ } ",262.32019999588374,,Completions generated successfully. Ready for display in your IDE.,68430855-e7e5-4f01-add1-7e9a1ebbbd00,1,deepseek-ai/deepseek-coder-1.3b-base," if (vowels.indexOf(s[i]) !== -1) { -",237,0.9545 +",237,0.9545 2025-09-22T13:57:50.008084,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13587,7 +13587,7 @@ class Problem { } }",121.5161999934935,,Completions generated successfully. Ready for display in your IDE.,803b8347-82e3-4432-87cb-75da2865a135,1,deepseek-ai/deepseek-coder-1.3b-base," fib += prev; -",96,1.0 +",96,1.0 2025-09-22T13:57:50.068978,[1],"//You are given two intervals, // where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). // The given intervals are closed which means that the interval (start, end) @@ -13616,7 +13616,7 @@ function intersection(interval1, interval2){ let length = end - start;"," return primes.includes(length) ? ""YES"" : ""NO""; } ",60.48449999798322,,Completions generated successfully. Ready for display in your IDE.,ed6858dc-9aa9-4240-ad3e-1a32e2acc7eb,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:50.528179,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13636,7 +13636,7 @@ class Problem { float mean = (float)numbers.stream().mapToDouble(Float::floatValue).average().getAsDouble();"," } }",459.2399999964982,,Completions generated successfully. Ready for display in your IDE.,b65d5385-b51a-49d9-939a-6a79c008389d,1,deepseek-ai/deepseek-coder-1.3b-base," return (float)(Math.abs(numbers.stream().mapToDouble(x -> Math.abs(x-mean)).average().getAsDouble())); -",411,0.9852 +",411,0.9852 2025-09-22T13:57:50.819456,[1],"from typing import List def sort_third(l: List[int]) -> List[int]: @@ -13654,7 +13654,7 @@ def sort_third(l: List[int]) -> List[int]: return l ",290.7844000001205,,Completions generated successfully. Ready for display in your IDE.,5d07ee89-4c39-4758-a503-63de8f772668,1,deepseek-ai/deepseek-coder-1.3b-base," l[j], l[j + 3] = l[j + 3], l[j] -",267,1.0 +",267,1.0 2025-09-22T13:57:50.935379,[1],"from typing import List def rolling_max(numbers: List[int]) -> List[int]: @@ -13668,7 +13668,7 @@ def rolling_max(numbers: List[int]) -> List[int]: return [max(numbers[:i + 1]) for i in range(len(numbers))] ",115.36730000079842,,Completions generated successfully. Ready for display in your IDE.,cbdee79b-ab9c-43da-9a95-7357b5a86c88,1,deepseek-ai/deepseek-coder-1.3b-base," if not numbers: -",83,0.9286 +",83,0.9286 2025-09-22T13:57:51.023791,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13697,7 +13697,7 @@ class Problem { } }",88.64419999736128,,Completions generated successfully. Ready for display in your IDE.,6a89f875-1f41-4b91-892f-01d7c8ad8633,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:51.201416,[1],"//Given a positive integer N, return the total sum of its digits in binary. // Example // >>> solve(1000) @@ -13720,7 +13720,7 @@ function solve(N){ return total.toString(2); }",177.0565999977407,,Completions generated successfully. Ready for display in your IDE.,27a75b57-f22f-41b6-8943-a547a7b6260c,1,deepseek-ai/deepseek-coder-1.3b-base," const numToString = (num).toString(); -",155,0.8308 +",155,0.8308 2025-09-22T13:57:51.397783,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -13792,7 +13792,7 @@ function minPath(grid, k){ return path; }",196.45470000250498,,Completions generated successfully. Ready for display in your IDE.,c1abf618-4f22-4801-a46f-72ae8db93d5a,1,deepseek-ai/deepseek-coder-1.3b-base," if(x < n - 1){ -",166,1.0 +",166,1.0 2025-09-22T13:57:52.100695,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13824,7 +13824,7 @@ class Problem { throw new IllegalArgumentException(""Invalid input""); } else if (!(k > 0)) { - for (int",677,0.8953 + for (int",677,0.8953 2025-09-22T13:57:52.165428,[1],"//Given the lengths of the three sides of a triangle. Return true if the three // sides form a right-angled triangle, false otherwise. // A right-angled triangle is a triangle in which one angle is right angle or @@ -13840,7 +13840,7 @@ function right_angle_triangle(a, b, c){ } else { return false; }",,63.92059999780031,,Completions generated successfully. Ready for display in your IDE.,49d11ac7-e096-4b32-9f1e-7810746013ae,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:57:52.269807,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -13852,7 +13852,7 @@ def all_prefixes(string: str) -> List[str]: prefixes.append(string[:i+1]) return prefixes ",104.45189999882132,,Completions generated successfully. Ready for display in your IDE.,7bdbe105-ef84-4cca-84e0-37c071f31775,1,deepseek-ai/deepseek-coder-1.3b-base," prefixes = [] -",81,1.0 +",81,1.0 2025-09-22T13:57:52.447371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13920,7 +13920,7 @@ class Problem { } }",177.6398000001791,,Completions generated successfully. Ready for display in your IDE.,b8401951-ebcd-490a-8568-75bbbd9e4ccd,1,deepseek-ai/deepseek-coder-1.3b-base," long running_max = 0; -",146,1.0 +",146,1.0 2025-09-22T13:57:52.500409,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -13947,7 +13947,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: return True return False ",52.79129999689758,,Completions generated successfully. Ready for display in your IDE.,c7f4fd1c-ee17-4188-aab9-6b817bb096af,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:57:52.541524,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -13970,7 +13970,7 @@ def sort_array(array: List[int]) -> List[int]: [6, 5, 4, 3, 2, 1, 0] """""" "," return sorted(array, reverse=True) if (array[0] + array[-1]) % 2 == 0 else sorted(array) - return array",40.960599995742086,,Completions generated successfully. Ready for display in your IDE.,d2e42fd0-a1a7-4e12-a030-cee097c7b759,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 + return array",40.960599995742086,,Completions generated successfully. Ready for display in your IDE.,d2e42fd0-a1a7-4e12-a030-cee097c7b759,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:57:53.256359,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -14043,7 +14043,7 @@ function do_algebra(operator, operand){ do_again = true; break; case '/': - operand[count] = operand[count] /",687,0.9801 + operand[count] = operand[count] /",687,0.9801 2025-09-22T13:57:53.310919,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14072,7 +14072,7 @@ class Problem { return newString.toString(); } }",53.89229999855161,,Completions generated successfully. Ready for display in your IDE.,daf74761-e8e9-4f46-969e-e947ba060d53,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,0.9821 +",31,0.9821 2025-09-22T13:57:53.528305,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14112,7 +14112,7 @@ class Problem { } }",217.32340000016848,,Completions generated successfully. Ready for display in your IDE.,1dddda31-80fd-4fd3-9734-483afcccf323,1,deepseek-ai/deepseek-coder-1.3b-base," Set uniqueCharacters = new HashSet<>(); -",192,1.0 +",192,1.0 2025-09-22T13:57:53.625183,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14156,7 +14156,7 @@ class Problem { } }",96.07279999909224,,Completions generated successfully. Ready for display in your IDE.,d59ef2af-890d-4a90-90c6-f83b9b677aa6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:53.706437,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14223,7 +14223,7 @@ class Problem { } } }",81.68919999297941,,Completions generated successfully. Ready for display in your IDE.,05495bd2-d087-4042-bfb0-75cf84e8eaa0,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:57:53.981005,[1],"from typing import List def sort_array(arr: List[int]) -> List[int]: @@ -14240,7 +14240,7 @@ def sort_array(arr: List[int]) -> List[int]: >>> sort_array([1, 0, 2, 3, 4]) [0, 1, 2, 3, 4] """""" -",,273.7691000002087,,Completions generated successfully. Ready for display in your IDE.,2777b5e0-9b50-4226-b523-2e59b3cc472d,1,deepseek-ai/deepseek-coder-1.3b-base," return sorted(arr, key=lambda x: (bin(x).count('1'), x))",251,0.9968 +",,273.7691000002087,,Completions generated successfully. Ready for display in your IDE.,2777b5e0-9b50-4226-b523-2e59b3cc472d,1,deepseek-ai/deepseek-coder-1.3b-base," return sorted(arr, key=lambda x: (bin(x).count('1'), x))",251,0.9968 2025-09-22T13:57:54.158807,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14288,7 +14288,7 @@ class Problem { } }",178.18830000032904,,Completions generated successfully. Ready for display in your IDE.,67817959-8005-47f5-987d-182828a91113,1,deepseek-ai/deepseek-coder-1.3b-base," indexMin = indexPlanet2; -",149,1.0 +",149,1.0 2025-09-22T13:57:54.347476,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14327,7 +14327,7 @@ class Problem { } }",188.20300000516,,Completions generated successfully. Ready for display in your IDE.,d4a4f1a4-69d1-46cd-b845-7ebfae0b8275,1,deepseek-ai/deepseek-coder-1.3b-base," if (currSum < minSum) { -",162,1.0 +",162,1.0 2025-09-22T13:57:54.459384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14374,7 +14374,7 @@ class Problem { } }",111.82900000130758,,Completions generated successfully. Ready for display in your IDE.,e0488f7c-c41c-45a5-a4a7-24bc317dea6a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:57:54.646888,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14412,7 +14412,7 @@ class Problem { }",186.74069999542553,,Completions generated successfully. Ready for display in your IDE.,7338edaf-6a3e-474e-9586-b18ccb2172a4,1,deepseek-ai/deepseek-coder-1.3b-base," temp = new StringBuffer(); } -",162,0.9957 +",162,0.9957 2025-09-22T13:57:54.712015,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14435,7 +14435,7 @@ class Problem { } return greatestCommonDivisor(b, a % b); }",,65.39199999679113,,Completions generated successfully. Ready for display in your IDE.,baa19ba7-0de1-474c-b574-bc23214193fa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:57:55.032333,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14483,7 +14483,7 @@ class Problem { } }",319.84790000569774,,Completions generated successfully. Ready for display in your IDE.,fe36cf7f-4bfa-41d0-8684-017e096f4fda,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = indexMin + 1; i <= indexMax - 1; ++i) { -",291,0.9946 +",291,0.9946 2025-09-22T13:57:55.191598,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14561,7 +14561,7 @@ class Problem { } }",159.1757999995025,,Completions generated successfully. Ready for display in your IDE.,bad9a35b-1336-4e9e-b7c1-7ab669863562,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 900; -",133,1.0 +",133,1.0 2025-09-22T13:57:55.463797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14592,7 +14592,7 @@ class Problem { while (x % i == 0) { x /= i; } -",247,1.0 +",247,1.0 2025-09-22T13:57:55.561797,[1],"//Given a string s and a natural number n, you have been tasked to implement // a function that returns an array of all words from string s that contain exactly // n consonants, in order these words appear in the string s. @@ -14618,7 +14618,7 @@ function select_words(s, n){ } ",97.99309999652905,,Completions generated successfully. Ready for display in your IDE.,713573c4-f90e-4421-a4a0-1c84a0395cac,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",73,1.0 +",73,1.0 2025-09-22T13:57:55.634951,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14658,7 +14658,7 @@ class Problem { } return arrayList; }",,72.3805999950855,,Completions generated successfully. Ready for display in your IDE.,25e5addd-d9ce-44dd-8f16-753a80d9e72b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:57:56.207224,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14685,7 +14685,7 @@ class Problem { Float nextNum = numbers.get(i + 1); double diff = Math.abs(nextNum - num); if (diff <= threshold) { -",546,0.9678 +",546,0.9678 2025-09-22T13:57:56.317882,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -14714,7 +14714,7 @@ function split_words(txt){ }"," } }",110.28660000010859,,Completions generated successfully. Ready for display in your IDE.,bd2be2dd-a323-4044-a84d-58b9b85a16f6,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",86,1.0 +",86,1.0 2025-09-22T13:57:56.455558,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -14738,7 +14738,7 @@ function parse_music(music_string){ } ",137.0803999961936,,Completions generated successfully. Ready for display in your IDE.,32811252-6413-4b86-b192-c3c1faeec0b6,1,deepseek-ai/deepseek-coder-1.3b-base," song.push(1); -",114,1.0 +",114,1.0 2025-09-22T13:57:56.548141,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -14756,7 +14756,7 @@ function parse_music(music_string){ factor += 1"," ",92.722400004277,,Completions generated successfully. Ready for display in your IDE.,8f0b3c6c-5ea6-461b-a437-e60715466752,1,deepseek-ai/deepseek-coder-1.3b-base," return largest -",72,0.9431 +",72,0.9431 2025-09-22T13:57:56.640063,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -14775,7 +14775,7 @@ function sum_product(numbers){ } ",91.81270000408404,,Completions generated successfully. Ready for display in your IDE.,8f1fa288-6426-4ee4-8baf-ecc8766370af,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",70,0.9003 +",70,0.9003 2025-09-22T13:57:56.713560,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -14813,7 +14813,7 @@ function exchange(lst1, lst2){ } } ",73.538999997254,,Completions generated successfully. Ready for display in your IDE.,86fdd628-0492-4b76-9244-025d09c5f26e,1,deepseek-ai/deepseek-coder-1.3b-base," - ",49,1.0 + ",49,1.0 2025-09-22T13:57:56.824643,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -14855,7 +14855,7 @@ function compare_one(a, b){ } ",110.28090000036173,,Completions generated successfully. Ready for display in your IDE.,92cc05b0-b26d-4a74-8c44-8d28d98b2d75,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",86,1.0 +",86,1.0 2025-09-22T13:57:56.941170,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14923,7 +14923,7 @@ class Problem { } }",116.88620000495575,,Completions generated successfully. Ready for display in your IDE.,663c516d-bfdd-47c1-9632-3e787ec12e3c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",84,1.0 +",84,1.0 2025-09-22T13:57:57.056599,[1],"//Given an array of integers nums, find the minimum sum of any non-empty sub-array // of nums. // Example @@ -14945,7 +14945,7 @@ function minSubArraySum(nums){ } ",114.85130000073696,,Completions generated successfully. Ready for display in your IDE.,5accda74-f719-448e-b80d-6308d842ecea,1,deepseek-ai/deepseek-coder-1.3b-base," sum = 0; -",93,1.0 +",93,1.0 2025-09-22T13:57:57.157285,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14989,7 +14989,7 @@ class Problem { } }",100.24189999967348,,Completions generated successfully. Ready for display in your IDE.,2eb24595-c233-482f-8226-0535121f9183,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:57:57.350729,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15042,7 +15042,7 @@ class Problem { } }",192.92110000242246,,Completions generated successfully. Ready for display in your IDE.,b7e02d22-d6b3-4e69-ad79-3a33c9500391,1,deepseek-ai/deepseek-coder-1.3b-base," if (minIndex == -1) { -",162,1.0 +",162,1.0 2025-09-22T13:57:57.480221,[1],"//You're a hungry rabbit, and you already have eaten a certain number of carrots, // but now you need to eat more carrots to complete the day's meals. // you should return an array of [ total number of eaten carrots after your meals, @@ -15073,7 +15073,7 @@ function eat(number, need, remaining){ "," else return [number + need, remaining - need]; } ",130.083600000944,,Completions generated successfully. Ready for display in your IDE.,230fb330-c2e9-4f2f-bcdb-5aacbb00b55e,1,deepseek-ai/deepseek-coder-1.3b-base," //your code here... - ",105,0.8994 + ",105,0.8994 2025-09-22T13:57:57.604748,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -15108,7 +15108,7 @@ function eat(number, need, remaining){ if e_name not in ['txt', 'exe', 'dll']: return 'No'"," ",123.99530000402592,,Completions generated successfully. Ready for display in your IDE.,2c3a4b0c-62af-4a65-81a2-5c6a3568fb96,1,deepseek-ai/deepseek-coder-1.3b-base," - return 'Yes'",99,0.9556 + return 'Yes'",99,0.9556 2025-09-22T13:57:57.830431,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -15137,7 +15137,7 @@ function f(n){ return result; }",225.43460000451887,,Completions generated successfully. Ready for display in your IDE.,c2a5da7e-106c-46ab-a975-df27758bac29,1,deepseek-ai/deepseek-coder-1.3b-base," for(var k = 1; k <= i; k++){ -",199,1.0 +",199,1.0 2025-09-22T13:57:57.952905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15181,7 +15181,7 @@ class Problem { }).count(); } }",122.04880000354024,,Completions generated successfully. Ready for display in your IDE.,b840730b-55cd-401f-a077-e81fa8e6f964,1,deepseek-ai/deepseek-coder-1.3b-base," class helper { -",79,0.994 +",79,0.994 2025-09-22T13:57:58.156381,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15213,7 +15213,7 @@ class Problem { } }",203.02999999694293,,Completions generated successfully. Ready for display in your IDE.,58462354-9588-4ddf-a1b5-f5c18e173fd7,1,deepseek-ai/deepseek-coder-1.3b-base," for (String paren:paren_strings) { -",177,1.0 +",177,1.0 2025-09-22T13:57:58.423628,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15247,7 +15247,7 @@ class Problem { } }",267.48879999649944,,Completions generated successfully. Ready for display in your IDE.,0c21915c-4491-4f99-96a1-db6e221c1633,1,deepseek-ai/deepseek-coder-1.3b-base," if ((fib & 1) == 0 && isPrime(fib)) { -",242,0.9618 +",242,0.9618 2025-09-22T13:57:58.518957,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -15302,7 +15302,7 @@ function by_length(arr){ return arr3; }",95.15790000295965,,Completions generated successfully. Ready for display in your IDE.,3ec7e928-3714-423d-82cd-f4f1baf818b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:58.608792,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15332,7 +15332,7 @@ class Problem { } return newString.toString();",},89.62469999823952,,Completions generated successfully. Ready for display in your IDE.,344eed9e-544e-4f22-8674-1b18e7ec7356,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:59.244877,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -15378,7 +15378,7 @@ function get_odd_collatz(n){ // console.log('Odd Numbers', odd_numbers); if (collatz_sequence[i] === 1 ) { -",608,0.9441 +",608,0.9441 2025-09-22T13:57:59.329512,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -15416,7 +15416,7 @@ function find_max(words){ } return max_unique_word; }",84.69109999714419,,Completions generated successfully. Ready for display in your IDE.,fd328ccc-7139-476c-a04a-fd5ac30b139c,1,deepseek-ai/deepseek-coder-1.3b-base,"else { -",59,0.9714 +",59,0.9714 2025-09-22T13:57:59.393453,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -15449,7 +15449,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.64929999836022,,Completions generated successfully. Ready for display in your IDE.,65abea14-38d6-4e8d-8f20-b21e4286def6,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:59.436515,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -15467,7 +15467,7 @@ def below_zero(operations: List[int]) -> bool: if balance < 0: return True"," print(below_zero([1, 2, 3])) -print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated successfully. Ready for display in your IDE.,c9128362-ab47-4c82-8c7e-2afa322be4a9,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.4378 +print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated successfully. Ready for display in your IDE.,c9128362-ab47-4c82-8c7e-2afa322be4a9,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.4378 2025-09-22T13:57:59.646985,[1],"//Given a positive floating point number, it can be decomposed into // and integer part (largest integer smaller than given number) and decimals // (leftover part always smaller than 1). @@ -15477,7 +15477,7 @@ print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated succe function truncate_number(number){ ","} ",210.6597000019974,,Completions generated successfully. Ready for display in your IDE.,8b634f6f-2f91-437f-96dc-c486bf8d990c,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.trunc(number); -",122,0.9595 +",122,0.9595 2025-09-22T13:57:59.792432,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15524,7 +15524,7 @@ class Problem { } }",144.91629999974975,,Completions generated successfully. Ready for display in your IDE.,f6167c7d-eb39-471a-8f0b-8b4c318d7027,1,deepseek-ai/deepseek-coder-1.3b-base," count += units; -",115,1.0 +",115,1.0 2025-09-22T13:57:59.921927,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -15554,7 +15554,7 @@ function sort_third(l){ return x; }",128.8979999953881,,Completions generated successfully. Ready for display in your IDE.,4ccf9eb3-bd0b-404f-86bd-87d61ef26ccc,1,deepseek-ai/deepseek-coder-1.3b-base," let k=0; -",103,1.0 +",103,1.0 2025-09-22T13:58:00.011389,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -15580,7 +15580,7 @@ function fibfib(n){ ",89.90990000165766,,Completions generated successfully. Ready for display in your IDE.,8163c0a2-b7e1-481f-8aab-d4c381c1a93c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T13:58:00.110085,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -15622,7 +15622,7 @@ function is_sorted(lst){ } ",98.23740000138059,,Completions generated successfully. Ready for display in your IDE.,d5a4e719-aa76-49b5-996e-9c9547a72b79,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:58:00.813493,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -15653,7 +15653,7 @@ function triples_sum_to_zero(l){ var a = [...new Set((Array).concat(l))]; console.log(""a"", a); - for (let x=0 ;x>> median([3, 1, 2, 4, 5]) // 3 @@ -15836,7 +15836,7 @@ function median(l){ } else { return sorted[middle]; }",,61.20370000280673,,Completions generated successfully. Ready for display in your IDE.,cdaa4a54-a3c1-455a-84f0-27c3d915079b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:01.364529,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -15885,7 +15885,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",58.18900000303984,,Completions generated successfully. Ready for display in your IDE.,a7ee50b9-b8e2-4125-8a42-5f40afd34fc8,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T13:58:01.426050,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -15928,7 +15928,7 @@ function histogram(test){ return result; } ",61.01840000337688,,Completions generated successfully. Ready for display in your IDE.,9c419bc2-bab9-4268-b01c-4756be381a72,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:58:01.490021,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15959,7 +15959,7 @@ class Problem { return sum; } }",63.599900000554044,,Completions generated successfully. Ready for display in your IDE.,cd9bd909-030d-4b81-ac2b-03d0915b5265,1,deepseek-ai/deepseek-coder-1.3b-base," -",40,1.0 +",40,1.0 2025-09-22T13:58:01.657245,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15981,7 +15981,7 @@ class Problem { } }",166.9605999995838,,Completions generated successfully. Ready for display in your IDE.,eeb26d75-4862-4b3a-a6b0-77cf87afd7f1,1,deepseek-ai/deepseek-coder-1.3b-base," sum += lst.get(i); -",145,1.0 +",145,1.0 2025-09-22T13:58:01.817003,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -16039,7 +16039,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.71469999931287,,Completions generated successfully. Ready for display in your IDE.,f4143e84-a898-4b52-92d2-d4de3e5e558e,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 2.3: -",131,1.0 +",131,1.0 2025-09-22T13:58:01.917162,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -16083,7 +16083,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",99.94749999896158,,Completions generated successfully. Ready for display in your IDE.,6d325a6b-73bf-4ad4-983b-0d00cbf68d8f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,0.8445 +",71,0.8445 2025-09-22T13:58:02.029301,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -16155,7 +16155,7 @@ function minPath(grid, k){ currCell = nextCell;"," return path; }",112.17999999644235,,Completions generated successfully. Ready for display in your IDE.,69e6657d-3813-446a-a613-8245d985473f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:58:02.176519,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -16189,7 +16189,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",146.77110000047833,,Completions generated successfully. Ready for display in your IDE.,2070a93a-72d8-421a-956d-99518559d7c2,1,deepseek-ai/deepseek-coder-1.3b-base," temp2_index = j -",122,1.0 +",122,1.0 2025-09-22T13:58:02.241578,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -16216,7 +16216,7 @@ function solve(s){ return swapped.join(''); } return s.split('').reverse().join('');",,64.83370000205468,,Completions generated successfully. Ready for display in your IDE.,f008d747-1dd3-436d-b202-579936b2b0fa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:02.296913,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -16240,7 +16240,7 @@ function solve(s){ else: return False"," ",55.3622000006726,,Completions generated successfully. Ready for display in your IDE.,9b067314-ce87-408f-a067-159468f1bcbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",23,1.0 +",23,1.0 2025-09-22T13:58:02.570047,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -16266,7 +16266,7 @@ function string_xor(a, b){ return result; }",272.8852000000188,,Completions generated successfully. Ready for display in your IDE.,83754469-0bce-45ca-adaf-20d0d29c34d8,1,deepseek-ai/deepseek-coder-1.3b-base," if ((arrayA[k] + arrayB[k]) %2 === 0 ){ -",248,0.952 +",248,0.952 2025-09-22T13:58:02.633112,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16301,7 +16301,7 @@ class Problem { ArrayList ans = new ArrayList(Arrays.asList(odds_arr));"," return ans; } }",62.895600000047125,,Completions generated successfully. Ready for display in your IDE.,c040edc9-139b-4c7c-9d34-6e42e4c3fe66,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:02.957578,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -16320,7 +16320,7 @@ function triangle_area(a, b, c){ } ",323.93690000026254,,Completions generated successfully. Ready for display in your IDE.,a8cc557b-512d-4f33-9872-c42200110795,1,deepseek-ai/deepseek-coder-1.3b-base," console.log((s*(s-a)*(s-b)*(s-c))**(.5)); -",301,0.9794 +",301,0.9794 2025-09-22T13:58:03.020582,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -16360,7 +16360,7 @@ function int_to_mini_roman(number){ return output; } ",63.02609999693232,,Completions generated successfully. Ready for display in your IDE.,e19fe86c-56ea-4f97-9f54-2eb363dd08a9,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T13:58:03.185943,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""<"" and "">"". return True if every opening bracket has a corresponding closing bracket. @@ -16383,7 +16383,7 @@ function int_to_mini_roman(number){ if num_left_brackets < 0: return False"," ",165.1588000022457,,Completions generated successfully. Ready for display in your IDE.,dcc342eb-d63d-4450-b06f-e35fc726eb4a,1,deepseek-ai/deepseek-coder-1.3b-base," - return num_left_brackets == 0",142,1.0 + return num_left_brackets == 0",142,1.0 2025-09-22T13:58:03.344796,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -16419,7 +16419,7 @@ function int_to_mini_roman(number){ ",158.6825999984285,,Completions generated successfully. Ready for display in your IDE.,a3d37171-4ae1-4e33-b7a7-91cbadc2fe7a,1,deepseek-ai/deepseek-coder-1.3b-base," for i in file_name: -",131,1.0 +",131,1.0 2025-09-22T13:58:03.597649,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -16446,7 +16446,7 @@ function strange_sort_list(lst){ return output; }",252.47980000131065,,Completions generated successfully. Ready for display in your IDE.,780f4be0-03e3-401a-85e7-d4c55ed68b80,1,deepseek-ai/deepseek-coder-1.3b-base," tmp.splice(tmp.indexOf(output[i]), 1); -",227,0.8942 +",227,0.8942 2025-09-22T13:58:04.012763,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16477,7 +16477,7 @@ class Problem { } }",414.40090000105556,,Completions generated successfully. Ready for display in your IDE.,eb4e07e2-a6da-4755-820b-69088b4e1b57,1,deepseek-ai/deepseek-coder-1.3b-base," boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$"")); -",386,1.0 +",386,1.0 2025-09-22T13:58:04.491313,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -16514,7 +16514,7 @@ function search(lst){ //create hash table with key as numbers from list values and vals as their frequencies for (var i=0; i < lst.length; i ++) { -",454,0.9176 +",454,0.9176 2025-09-22T13:58:04.585330,[1],"from typing import List, Tuple def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: @@ -16539,7 +16539,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: res = [(i, j) for i in range(len(lst)) for j in range(len(lst[i])) if lst[i][j] == x] res.sort(key=lambda t: (t[0], -t[1]))"," ",94.50729999662144,,Completions generated successfully. Ready for display in your IDE.,cb4e2b6b-8dd5-4838-aef9-27a4e3cf67c7,1,deepseek-ai/deepseek-coder-1.3b-base," - return res",69,1.0 + return res",69,1.0 2025-09-22T13:58:04.688030,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -16563,7 +16563,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: return string ",102.22950000024866,,Completions generated successfully. Ready for display in your IDE.,24d49b6e-636c-4a47-8b92-f101c7d84767,1,deepseek-ai/deepseek-coder-1.3b-base," if not string: -",79,1.0 +",79,1.0 2025-09-22T13:58:04.802864,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -16601,7 +16601,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",114.57959999825107,,Completions generated successfully. Ready for display in your IDE.,fcb60b0e-eac0-4eb5-9c37-5cb5d2a2a446,1,deepseek-ai/deepseek-coder-1.3b-base," even++; -",90,1.0 +",90,1.0 2025-09-22T13:58:04.893590,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -16615,7 +16615,7 @@ function filter_by_substring(strings, substring){ }"," return filtered; }",90.29430000373395,,Completions generated successfully. Ready for display in your IDE.,ea368f2b-21ab-4524-8f6c-684932aa2598,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:04.988363,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -16640,7 +16640,7 @@ function make_palindrome(string){ } ",95.03000000404427,,Completions generated successfully. Ready for display in your IDE.,4d92a830-9b1c-44cd-bed4-56e6d2ba199a,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",72,1.0 +",72,1.0 2025-09-22T13:58:05.199622,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -16668,7 +16668,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: return min2 if min1 != min2 else None ",210.81630000117002,,Completions generated successfully. Ready for display in your IDE.,a6341523-47c7-4009-8eb4-77fa47aa3d3e,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, len(lst)): -",185,1.0 +",185,1.0 2025-09-22T13:58:05.377038,[1],"from typing import List def max_element(l: List[int]) -> int: @@ -16683,7 +16683,7 @@ def max_element(l: List[int]) -> int: return max_elem ",177.56910000025528,,Completions generated successfully. Ready for display in your IDE.,3f14dfe2-1829-4e69-92c6-475408c184be,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): -",155,1.0 +",155,1.0 2025-09-22T13:58:05.494930,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -16712,7 +16712,7 @@ function f(n){ return result; }",116.97819999972126,,Completions generated successfully. Ready for display in your IDE.,bad39326-5e9a-4c76-98ec-751497f0f51b,1,deepseek-ai/deepseek-coder-1.3b-base," sum += k; -",93,1.0 +",93,1.0 2025-09-22T13:58:05.693330,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -16741,7 +16741,7 @@ function next_smallest(lst){ }else{ return second_smallest; } -",175,0.9757 +",175,0.9757 2025-09-22T13:58:05.810384,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -16771,7 +16771,7 @@ function parse_nested_parens(paren_string){ ",117.05229999643052,,Completions generated successfully. Ready for display in your IDE.,e2d9b26f-0ed8-459b-8bba-3513d4d8d769,1,deepseek-ai/deepseek-coder-1.3b-base," level += 1; -",93,1.0 +",93,1.0 2025-09-22T13:58:05.924341,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -16813,7 +16813,7 @@ function compare_one(a, b){ } ",113.2556000011391,,Completions generated successfully. Ready for display in your IDE.,b9270a89-5012-4446-a773-1b18b83a5e90,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",83,0.9914 +",83,0.9914 2025-09-22T13:58:06.066514,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""<"" and "">"". return True if every opening bracket has a corresponding closing bracket. @@ -16837,7 +16837,7 @@ function compare_one(a, b){ ",142.09739999932935,,Completions generated successfully. Ready for display in your IDE.,a73ba4b2-9ff3-45d5-8506-4314503a2928,1,deepseek-ai/deepseek-coder-1.3b-base," if bracket == '<': -",120,1.0 +",120,1.0 2025-09-22T13:58:06.171919,[1],"//Given a non-empty array of integers lst. add the even elements that are at odd indices.. // Examples: // >>> add([4, 2, 6, 7]) @@ -16850,7 +16850,7 @@ function add(lst){ } }",},105.07179999694927,,Completions generated successfully. Ready for display in your IDE.,35adc72d-77c3-41cf-b2d4-550637416bfe,1,deepseek-ai/deepseek-coder-1.3b-base," return add; -",84,1.0 +",84,1.0 2025-09-22T13:58:06.333595,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16879,7 +16879,7 @@ class Problem { });"," } }",161.34880000026897,,Completions generated successfully. Ready for display in your IDE.,b13b7098-2415-4bee-b33f-e313b47c354b,1,deepseek-ai/deepseek-coder-1.3b-base," return String.join("" "", split); -",136,0.9922 +",136,0.9922 2025-09-22T13:58:06.511736,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16924,7 +16924,7 @@ class Problem { } }",178.0750000034459,,Completions generated successfully. Ready for display in your IDE.,05915032-b905-49c3-a42c-7b2101410b00,1,deepseek-ai/deepseek-coder-1.3b-base," ans.append(word + "" ""); -",152,0.9615 +",152,0.9615 2025-09-22T13:58:06.600428,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -16955,7 +16955,7 @@ function total_match(lst1, lst2){ } ",88.44210000097519,,Completions generated successfully. Ready for display in your IDE.,27b8cdb6-b9a4-458f-bb04-a71718e13275,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:06.719961,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -16978,7 +16978,7 @@ function encode(message){ return result; }",119.50390000129119,,Completions generated successfully. Ready for display in your IDE.,ee9b0efd-d9db-4331-a244-8f200ba04f12,1,deepseek-ai/deepseek-coder-1.3b-base," var result= """"; -",93,0.951 +",93,0.951 2025-09-22T13:58:07.412064,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17018,7 +17018,7 @@ class Problem { private static final int[] primes = {7,9}; /** Returns the smallest number greater than n that is both prime and divisible by all of the numbers in the array. */ - public static long nextPrimeDivBy",667,0.8502 + public static long nextPrimeDivBy",667,0.8502 2025-09-22T13:58:07.558078,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -17040,7 +17040,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: else: odd_count += 1"," ",146.13440000539413,,Completions generated successfully. Ready for display in your IDE.,cdce061f-dbca-47f7-b312-564ca492caba,1,deepseek-ai/deepseek-coder-1.3b-base," - return even_count, odd_count",124,0.9915 + return even_count, odd_count",124,0.9915 2025-09-22T13:58:07.674222,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -17079,7 +17079,7 @@ function valid_date(date){ } ",115.81909999949858,,Completions generated successfully. Ready for display in your IDE.,9682ff18-5e0b-477f-9343-0305a1e923ab,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",89,1.0 +",89,1.0 2025-09-22T13:58:07.825086,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -17105,7 +17105,7 @@ function fibfib(n){ ",150.68189999874448,,Completions generated successfully. Ready for display in your IDE.,a5a9b58d-f871-4186-9026-d167d0affcb7,1,deepseek-ai/deepseek-coder-1.3b-base," throw new Error(""Invalid input""); -",124,0.9798 +",124,0.9798 2025-09-22T13:58:07.904502,[1],"//Given a non-empty array of integers, return the sum of all of the odd elements that are in even positions. // Examples // >>> solution([5, 8, 7, 1]) @@ -17124,7 +17124,7 @@ function solution(lst){ } } return result;",,79.30389999819454,,Completions generated successfully. Ready for display in your IDE.,2ba674fb-15e5-4633-a3d4-60383c567fa1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",53,1.0 +}",53,1.0 2025-09-22T13:58:08.058895,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -17143,7 +17143,7 @@ function is_bored(S){ } ",153.64250000129687,,Completions generated successfully. Ready for display in your IDE.,80cca7ce-023c-4098-b390-57cedfabb518,1,deepseek-ai/deepseek-coder-1.3b-base," for (let line in lines) { -",134,0.9654 +",134,0.9654 2025-09-22T13:58:08.321977,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17179,14 +17179,14 @@ class Problem { } }",262.7929999944172,,Completions generated successfully. Ready for display in your IDE.,62e4a4f4-019c-4304-b5ac-081b62fc9606,1,deepseek-ai/deepseek-coder-1.3b-base," if (lst2.get(j) % 2 == 1) { -",235,0.9811 +",235,0.9811 2025-09-22T13:58:08.371968,[1],"//Return sorted unique elements in an array // >>> unique([5, 3, 5, 2, 3, 3, 9, 0, 123]) // [0, 2, 3, 5, 9, 123] function unique(l){ return l.filter(function(e, i, a){ return a.indexOf(e) === i; - }).sort(function(a, b){return a-b});",,49.85599999781698,,Completions generated successfully. Ready for display in your IDE.,f0936ff0-36f3-4b3e-8b50-9e9d89f7b4d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.1946 + }).sort(function(a, b){return a-b});",,49.85599999781698,,Completions generated successfully. Ready for display in your IDE.,f0936ff0-36f3-4b3e-8b50-9e9d89f7b4d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.1946 2025-09-22T13:58:08.568550,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -17258,7 +17258,7 @@ function minPath(grid, k){ return path; }",196.50609999371227,,Completions generated successfully. Ready for display in your IDE.,1270fd65-1483-40ad-b840-c8d0430fd2e9,1,deepseek-ai/deepseek-coder-1.3b-base," if(y < n - 1){ -",167,1.0 +",167,1.0 2025-09-22T13:58:08.743298,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17305,7 +17305,7 @@ class Problem { } }",174.8321000050055,,Completions generated successfully. Ready for display in your IDE.,46b76a53-63dc-4430-b985-3e5087335d32,1,deepseek-ai/deepseek-coder-1.3b-base," long units = level / capacity; -",146,0.9545 +",146,0.9545 2025-09-22T13:58:08.837489,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17336,7 +17336,7 @@ class Problem { boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$"")); return allLowerCase || allUpperCase;",},93.60680000099819,,Completions generated successfully. Ready for display in your IDE.,9e9cd8f7-4f77-4fd3-9908-38943a10ada4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:08.971252,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -17359,7 +17359,7 @@ function factorize(n){ } ",133.80659999529598,,Completions generated successfully. Ready for display in your IDE.,8cbf1f87-2ad8-45a4-b9ea-b9dc3ff2db9c,1,deepseek-ai/deepseek-coder-1.3b-base," n /= factor; -",108,1.0 +",108,1.0 2025-09-22T13:58:09.067953,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -17387,7 +17387,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: if count > 0: return ""NO"""," return ""YES""",96.20699999504723,,Completions generated successfully. Ready for display in your IDE.,01f3a789-81d1-403a-b76f-141dbe4598e7,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:58:09.761170,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -17425,7 +17425,7 @@ function same_chars(s0, s1){ break; } - //loop through each character in string",669,0.9419 + //loop through each character in string",669,0.9419 2025-09-22T13:58:09.848279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17455,7 +17455,7 @@ class Problem { } return res;",},86.0259999972186,,Completions generated successfully. Ready for display in your IDE.,ca0b785d-e1ee-4c5e-aabe-03c4e0ed3d8b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:10.145066,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17486,7 +17486,7 @@ class Problem { while (x % i == 0) { primes.add(i); } -",270,1.0 +",270,1.0 2025-09-22T13:58:10.240652,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -17535,7 +17535,7 @@ function check_dict_case(dict){ return true; }",95.42570000485284,,Completions generated successfully. Ready for display in your IDE.,d8c5fccc-e19c-48dd-8e79-43b9a3498362,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9511 +",69,0.9511 2025-09-22T13:58:10.347324,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -17555,7 +17555,7 @@ function unique_digits(x){ } ",106.18790000444278,,Completions generated successfully. Ready for display in your IDE.,20eb3dd0-5480-4e7c-97de-0449f05b6124,1,deepseek-ai/deepseek-coder-1.3b-base," var result = []; -",80,0.9892 +",80,0.9892 2025-09-22T13:58:10.785568,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -17583,7 +17583,7 @@ function string_xor(a, b){ //convert the input to an integer value using parseint() method: for (var i = 0; i < a.length; i++) { -",415,0.953 +",415,0.953 2025-09-22T13:58:10.878210,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17615,7 +17615,7 @@ class Problem { } }",92.65210000012303,,Completions generated successfully. Ready for display in your IDE.,b2b55878-0bc8-4eb1-84a9-994b73ccf5b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:10.998899,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17654,7 +17654,7 @@ class Problem { }"," } }",120.72969999280758,,Completions generated successfully. Ready for display in your IDE.,eb9dee46-5cbf-4f58-80e6-e2f49972304c,1,deepseek-ai/deepseek-coder-1.3b-base," return minSum; -",100,1.0 +",100,1.0 2025-09-22T13:58:11.160011,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17686,7 +17686,7 @@ class Problem { } }",160.93230000115,,Completions generated successfully. Ready for display in your IDE.,45014262-fcee-499c-96d9-11c2e2aad646,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(0L); -",138,1.0 +",138,1.0 2025-09-22T13:58:11.316727,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -17741,7 +17741,7 @@ function by_length(arr){ return arr3; }",156.2476000035531,,Completions generated successfully. Ready for display in your IDE.,15af05e2-384d-4014-b4be-7abb4401a77e,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Four""); -",130,1.0 +",130,1.0 2025-09-22T13:58:11.472203,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -17783,7 +17783,7 @@ def move_one_ball(arr: List[int]) -> bool: ",155.63880000263453,,Completions generated successfully. Ready for display in your IDE.,aab48e06-36d5-49b6-8bdf-0afa7d29a233,1,deepseek-ai/deepseek-coder-1.3b-base," return False else: - ",131,0.9597 + ",131,0.9597 2025-09-22T13:58:11.948072,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17815,7 +17815,7 @@ class Problem { } }",475.33739999926183,,Completions generated successfully. Ready for display in your IDE.,27ac147c-fbee-4a95-a3ce-5dca144f5e37,1,deepseek-ai/deepseek-coder-1.3b-base," if (!text.substring(left, left + 1).equalsIgnoreCase(text.substring(right, right + 1))) { -",349,0.9731 +",349,0.9731 2025-09-22T13:58:12.144784,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17883,7 +17883,7 @@ class Problem { } }",196.75640000059502,,Completions generated successfully. Ready for display in your IDE.,f0ab80a1-1284-4884-9991-8617c94475c9,1,deepseek-ai/deepseek-coder-1.3b-base," for (Long number : lst) { -",167,0.9708 +",167,0.9708 2025-09-22T13:58:12.391777,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17912,7 +17912,7 @@ class Problem { } }",246.07010000181617,,Completions generated successfully. Ready for display in your IDE.,ac18328a-5878-4c59-887e-82c7b79a7f41,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(split, new Comparator() { -",222,1.0 +",222,1.0 2025-09-22T13:58:12.433788,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -17927,7 +17927,7 @@ function count_distinct_characters(string){ } return result.length; } -",41.9257999965339,,Completions generated successfully. Ready for display in your IDE.,8e36385e-8537-486c-b441-5c5ce7b7707d,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",41.9257999965339,,Completions generated successfully. Ready for display in your IDE.,8e36385e-8537-486c-b441-5c5ce7b7707d,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:58:12.583158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17989,7 +17989,7 @@ class Problem { } }",149.50690000114264,,Completions generated successfully. Ready for display in your IDE.,81222eff-5258-4431-b7af-08fc4a4c96bc,1,deepseek-ai/deepseek-coder-1.3b-base," String output = """"; -",122,0.9324 +",122,0.9324 2025-09-22T13:58:12.705009,[1],"from typing import List def incr_list(l: List[int]) -> List[int]: @@ -18003,7 +18003,7 @@ def incr_list(l: List[int]) -> List[int]: return res ",121.35729999863543,,Completions generated successfully. Ready for display in your IDE.,4de453a6-ecbd-4aab-aa3a-6499c912a98a,1,deepseek-ai/deepseek-coder-1.3b-base," for i in l: -",100,1.0 +",100,1.0 2025-09-22T13:58:12.975311,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18045,7 +18045,7 @@ class Problem { } }",270.20319999428466,,Completions generated successfully. Ready for display in your IDE.,75a12142-b74f-4c99-a99f-fdbf966a94b7,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < s4.length(); i++) { -",244,1.0 +",244,1.0 2025-09-22T13:58:13.153864,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18089,7 +18089,7 @@ class Problem { } }",177.95199999818578,,Completions generated successfully. Ready for display in your IDE.,e1f644f6-d2cb-4346-aebd-59f0e032f882,1,deepseek-ai/deepseek-coder-1.3b-base," if (currSum == 0) { -",152,1.0 +",152,1.0 2025-09-22T13:58:13.537631,[1],"from typing import List, Tuple def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: @@ -18115,7 +18115,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: ",383.5366999992402,,Completions generated successfully. Ready for display in your IDE.,b284f55d-4dfe-4d56-9280-bd5794b753ec,1,deepseek-ai/deepseek-coder-1.3b-base," res.sort(key=lambda x: (-x[0], len(lst)-x[0]-1-x[1])) -",359,0.9674 +",359,0.9674 2025-09-22T13:58:13.737420,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18146,7 +18146,7 @@ class Problem { boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$""));"," } }",199.4535999983782,,Completions generated successfully. Ready for display in your IDE.,98825a6e-3339-4f5a-b826-12227bc3fd49,1,deepseek-ai/deepseek-coder-1.3b-base," return allLowerCase || allUpperCase; -",168,1.0 +",168,1.0 2025-09-22T13:58:13.864132,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18179,7 +18179,7 @@ class Problem { }"," } }",126.23890000395477,,Completions generated successfully. Ready for display in your IDE.,e13397e9-d629-407e-8668-850c44693230,1,deepseek-ai/deepseek-coder-1.3b-base," return balance == 0; -",103,1.0 +",103,1.0 2025-09-22T13:58:13.989013,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -18227,7 +18227,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",125.18180000188295,,Completions generated successfully. Ready for display in your IDE.,f4c6f76f-ba57-43e5-9eac-33b1a8d5f72f,1,deepseek-ai/deepseek-coder-1.3b-base," row = i -",98,1.0 +",98,1.0 2025-09-22T13:58:14.095786,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -18259,7 +18259,7 @@ function correct_bracketing(brackets){ } }",106.51379999762867,,Completions generated successfully. Ready for display in your IDE.,838752e1-0414-4e8e-9ff3-943085bbeb4a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",84,1.0 +",84,1.0 2025-09-22T13:58:14.139474,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -18283,7 +18283,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: groups.append(group) group = '' return groups -",43.07939999853261,,Completions generated successfully. Ready for display in your IDE.,4ce9a8b7-f31f-4e3c-b957-4e470298b9a4,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.07939999853261,,Completions generated successfully. Ready for display in your IDE.,4ce9a8b7-f31f-4e3c-b957-4e470298b9a4,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:58:14.405502,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18316,7 +18316,7 @@ class Problem { } }",265.9630999987712,,Completions generated successfully. Ready for display in your IDE.,25dacfa5-b158-46df-93d9-d3bb57fee529,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < lst.size(); ++i) { -",242,0.9903 +",242,0.9903 2025-09-22T13:58:14.498833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18352,7 +18352,7 @@ class Problem { } return max;",},93.01899999991292,,Completions generated successfully. Ready for display in your IDE.,6bb63187-0b15-474c-b145-c8c8e0d92eb2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:14.583211,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -18368,7 +18368,7 @@ function rolling_max(numbers){ } ",84.81480000045849,,Completions generated successfully. Ready for display in your IDE.,0a1de400-6950-48f3-ae21-6c8d365f0ff4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:14.649732,[1],"//Return length of given string // >>> strlen("""") // 0 @@ -18376,7 +18376,7 @@ function rolling_max(numbers){ // 3 function strlen(string){ return string.length;",,65.53350000467617,,Completions generated successfully. Ready for display in your IDE.,bbb69aac-5c2e-4691-b888-423fa7b30553,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:14.791793,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -18393,7 +18393,7 @@ function largest_prime_factor(n){ return result; }",142.21670000551967,,Completions generated successfully. Ready for display in your IDE.,cc6f0156-6706-4028-882a-042ecce36e9c,1,deepseek-ai/deepseek-coder-1.3b-base," n /= result; -",120,1.0 +",120,1.0 2025-09-22T13:58:14.969116,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18423,7 +18423,7 @@ class Problem { return res; } }",177.1669000008842,,Completions generated successfully. Ready for display in your IDE.,4dc397a3-cb68-4d3c-8aa6-cbc496b9f39c,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList res = new ArrayList<>(); -",150,1.0 +",150,1.0 2025-09-22T13:58:15.013149,[1],"def fruit_distribution(s: str, n: int) -> int: """""" In this task, you will be given a string that represents a number of apples and oranges @@ -18449,7 +18449,7 @@ class Problem { return n - sum(fruits) -",43.967999998130836,,Completions generated successfully. Ready for display in your IDE.,ad0791cf-17f3-4db9-bb8b-9fe811ad30a6,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.8142 +",43.967999998130836,,Completions generated successfully. Ready for display in your IDE.,ad0791cf-17f3-4db9-bb8b-9fe811ad30a6,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.8142 2025-09-22T13:58:15.233095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18493,7 +18493,7 @@ class Problem { } }",219.434699996782,,Completions generated successfully. Ready for display in your IDE.,6ff4ddc3-0855-4855-85e6-b49a17952e90,1,deepseek-ai/deepseek-coder-1.3b-base," char left = word.charAt(j - 1); -",193,1.0 +",193,1.0 2025-09-22T13:58:15.341690,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -18522,7 +18522,7 @@ function is_simple_power(x, n){ } ",108.18730000028154,,Completions generated successfully. Ready for display in your IDE.,b7cf0bea-65a4-4aa3-9118-d9772a339297,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:58:15.435419,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18562,7 +18562,7 @@ class Problem { } }",93.86110000195913,,Completions generated successfully. Ready for display in your IDE.,1ed4c6dc-a215-44fb-9dfe-4493a6744b85,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:15.519682,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18611,7 +18611,7 @@ class Problem { } return valid_date; }",,83.77370000380324,,Completions generated successfully. Ready for display in your IDE.,d32cb07b-6421-4890-ade2-ffa7a17d6370,1,deepseek-ai/deepseek-coder-1.3b-base," -}",56,1.0 +}",56,1.0 2025-09-22T13:58:15.841993,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -18666,7 +18666,7 @@ function by_length(arr){ return arr3; }",322.1233000003849,,Completions generated successfully. Ready for display in your IDE.,d28e8d40-cc16-4b17-b9ea-ccc011e868d3,1,deepseek-ai/deepseek-coder-1.3b-base," if((typeof (num) == 'number') && ((num >= 1)&&(num <= 9))){ -",296,0.9472 +",296,0.9472 2025-09-22T13:58:16.051891,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18700,7 +18700,7 @@ class Problem { } }",210.05199999490287,,Completions generated successfully. Ready for display in your IDE.,855e9cba-5e79-48f6-a62f-f673af53f646,1,deepseek-ai/deepseek-coder-1.3b-base," char c = brackets.charAt(i); -",173,1.0 +",173,1.0 2025-09-22T13:58:16.292085,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -18736,7 +18736,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: else: sign_arr.append(0)",,240.03130000346573,,Completions generated successfully. Ready for display in your IDE.,32952b3b-085b-42bf-bcac-941fc86d7d78,1,deepseek-ai/deepseek-coder-1.3b-base," - return abs(product(arr)) * sum(sign_arr)",213,0.9902 + return abs(product(arr)) * sum(sign_arr)",213,0.9902 2025-09-22T13:58:16.751593,[1],"from typing import Tuple def even_odd_palindrome(n: int) -> Tuple[int, int]: @@ -18766,7 +18766,7 @@ def even_odd_palindrome(n: int) -> Tuple[int, int]: sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 == 0),"," ) ",459.36580000125105,,Completions generated successfully. Ready for display in your IDE.,20832017-9ff0-48ce-a78b-5d86963330d1,1,deepseek-ai/deepseek-coder-1.3b-base," sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 != 0) -",430,0.9968 +",430,0.9968 2025-09-22T13:58:16.825346,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18795,7 +18795,7 @@ class Problem { }); return String.join("" "", split); }",,73.37289999850327,,Completions generated successfully. Ready for display in your IDE.,a8e9343a-e501-414d-969f-dbb6ee93b12d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:58:16.923646,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -18823,7 +18823,7 @@ function monotonic(l){ } ",97.68930000427645,,Completions generated successfully. Ready for display in your IDE.,2ebed83f-6898-45d5-ae43-351d8696bc51,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:17.071470,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -18841,7 +18841,7 @@ function monotonic(l){ ",148.07099999597995,,Completions generated successfully. Ready for display in your IDE.,16ab4b38-7dd0-46ce-b4cd-03488c28e566,1,deepseek-ai/deepseek-coder-1.3b-base," if n % factor == 0: -",125,1.0 +",125,1.0 2025-09-22T13:58:17.159696,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18872,7 +18872,7 @@ class Problem { } }",87.51359999587294,,Completions generated successfully. Ready for display in your IDE.,6cfcc60c-e9a3-4ffa-8e02-fc8348f9baa9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:17.343847,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18919,7 +18919,7 @@ class Problem { } }",184.33509999886155,,Completions generated successfully. Ready for display in your IDE.,12819cad-98b4-47e3-9646-9e8a26e78d39,1,deepseek-ai/deepseek-coder-1.3b-base," for (long unit : well) { -",155,1.0 +",155,1.0 2025-09-22T13:58:17.479753,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -18969,7 +18969,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",135.127100002137,,Completions generated successfully. Ready for display in your IDE.,439af854-9c61-4325-98c5-65e19b02c64b,1,deepseek-ai/deepseek-coder-1.3b-base," return number > 1; -",108,1.0 +",108,1.0 2025-09-22T13:58:17.688733,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -19003,7 +19003,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",209.0978000051109,,Completions generated successfully. Ready for display in your IDE.,7a23b118-288d-47c0-9a8a-b2b8baf83aae,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(numbers)-1): -",182,1.0 +",182,1.0 2025-09-22T13:58:17.889684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19047,7 +19047,7 @@ class Problem { } }",200.48120000137715,,Completions generated successfully. Ready for display in your IDE.,5f9967c0-1b8b-4763-aeba-8d7015252ed6,1,deepseek-ai/deepseek-coder-1.3b-base," return digits.toArray(new Long[]{}); -",172,0.9962 +",172,0.9962 2025-09-22T13:58:18.039747,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19078,7 +19078,7 @@ class Problem { } }",150.18549999513198,,Completions generated successfully. Ready for display in your IDE.,c03e7c70-1d2a-4bf3-83fb-79c907895816,1,deepseek-ai/deepseek-coder-1.3b-base," while (temp > 0) { -",124,1.0 +",124,1.0 2025-09-22T13:58:18.149056,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -19111,7 +19111,7 @@ class Problem { ",108.79659999773139,,Completions generated successfully. Ready for display in your IDE.,d4ed1cbe-794a-43f8-bb4a-a38e4415e8b4,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",84,1.0 +",84,1.0 2025-09-22T13:58:18.231086,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19145,7 +19145,7 @@ class Problem { } return open == 0;",},81.60049999423791,,Completions generated successfully. Ready for display in your IDE.,660d79c8-0f85-4100-8ceb-4fb5a8816b0e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:58:18.367031,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -19174,7 +19174,7 @@ function fib4(n){ ",136.11539999692468,,Completions generated successfully. Ready for display in your IDE.,8da24113-e2f8-45e6-8868-d335f920f44b,1,deepseek-ai/deepseek-coder-1.3b-base," return prev[3]; -",112,0.8948 +",112,0.8948 2025-09-22T13:58:18.464411,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19219,7 +19219,7 @@ class Problem { } }",97.39429999899585,,Completions generated successfully. Ready for display in your IDE.,765ca340-6433-4a26-a4d2-2b75e22aafe6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:18.662935,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -19246,7 +19246,7 @@ def by_length(arr: List[int]) -> List[str]: return digits ",198.25990000390448,,Completions generated successfully. Ready for display in your IDE.,82ef2847-4304-4001-94fd-7d6e8c6ad81c,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(digits)): -",174,1.0 +",174,1.0 2025-09-22T13:58:18.841175,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -19288,7 +19288,7 @@ function compare_one(a, b){ } ",177.7224000034039,,Completions generated successfully. Ready for display in your IDE.,acb3b80d-49db-4c27-a502-3a33fb4a96d1,1,deepseek-ai/deepseek-coder-1.3b-base," if (a > b + """" ){ -",152,0.9242 +",152,0.9242 2025-09-22T13:58:18.940734,[1],"//sum_to_n is a function that sums numbers from 1 to n. // >>> sum_to_n(30) // 465 @@ -19307,7 +19307,7 @@ function sum_to_n(n){ }","} ",99.59890000027372,,Completions generated successfully. Ready for display in your IDE.,3f7cd686-5b89-4016-a016-0f04a6a32e3d,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",73,1.0 +",73,1.0 2025-09-22T13:58:19.224961,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19334,7 +19334,7 @@ class Problem { } } }",283.90000000217697,,Completions generated successfully. Ready for display in your IDE.,ca2376f1-1c3a-4298-a751-303545e90300,1,deepseek-ai/deepseek-coder-1.3b-base," if (n < 0 || !NumberUtils.isCreatable(Long.toString(n))) { -",257,0.9078 +",257,0.9078 2025-09-22T13:58:19.348896,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19357,7 +19357,7 @@ class Problem { return answer; } }",123.52429999737069,,Completions generated successfully. Ready for display in your IDE.,3cbafb53-6dcb-410c-8f9a-3a79f27af326,1,deepseek-ai/deepseek-coder-1.3b-base," String answer = """"; -",83,1.0 +",83,1.0 2025-09-22T13:58:19.453653,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19380,7 +19380,7 @@ class Problem { } }",104.25640000175918,,Completions generated successfully. Ready for display in your IDE.,b03e3f93-11d2-4fde-9bbd-98d57bf14ede,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",79,0.9171 +",79,0.9171 2025-09-22T13:58:19.863525,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19429,7 +19429,7 @@ class Problem { }",409.55290000420064,,Completions generated successfully. Ready for display in your IDE.,e39a891f-cbdc-4efb-9dd9-321777d9a0af,1,deepseek-ai/deepseek-coder-1.3b-base," Collections.sort(l_e); for (int i = 0; i < l_e.size(); i++) { -",379,1.0 +",379,1.0 2025-09-22T13:58:20.099332,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -19465,7 +19465,7 @@ class Problem { ",235.88320000271779,,Completions generated successfully. Ready for display in your IDE.,1a8c03b9-7c79-4242-a6a0-f62de9cfad1d,1,deepseek-ai/deepseek-coder-1.3b-base," e_name = file_name.split('.')[1] -",206,1.0 +",206,1.0 2025-09-22T13:58:20.194383,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19509,7 +19509,7 @@ class Problem { } }",94.68649999325862,,Completions generated successfully. Ready for display in your IDE.,64a6e405-8814-476b-b8a5-b8e80210c288,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:20.790109,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -19537,7 +19537,7 @@ function compare(game, guess){ const diff = Math.abs(guess[i]-game[i]); if (diff === 0 ){ -",570,0.9434 +",570,0.9434 2025-09-22T13:58:20.997926,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19568,7 +19568,7 @@ class Problem { int res_denom = x_denom * n_denom;"," } }",207.6874999984284,,Completions generated successfully. Ready for display in your IDE.,b44455a1-6e2a-4562-b23b-69814c061aa2,1,deepseek-ai/deepseek-coder-1.3b-base," return res_num % res_denom == 0; -",184,0.9852 +",184,0.9852 2025-09-22T13:58:21.159796,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19646,7 +19646,7 @@ class Problem { } }",161.12869999778923,,Completions generated successfully. Ready for display in your IDE.,09385545-063c-418a-9a90-e8a3fa43b94c,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 4) { -",134,1.0 +",134,1.0 2025-09-22T13:58:21.317605,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19669,7 +19669,7 @@ class Problem { } }",157.92859999783104,,Completions generated successfully. Ready for display in your IDE.,d7709e8c-c57a-4c23-8ef1-339ef22d0cca,1,deepseek-ai/deepseek-coder-1.3b-base," shift %= digits.length(); -",134,0.9364 +",134,0.9364 2025-09-22T13:58:21.465437,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -19736,7 +19736,7 @@ function numerical_letter_grade(grades){ } }",},147.3446000018157,,Completions generated successfully. Ready for display in your IDE.,8f7e9dc1-d93b-4c71-a709-06236a55d8ce,1,deepseek-ai/deepseek-coder-1.3b-base," return grades_array; -",119,1.0 +",119,1.0 2025-09-22T13:58:21.519239,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -19765,7 +19765,7 @@ function parse_nested_parens(paren_string){ } ",54.318599999533035,,Completions generated successfully. Ready for display in your IDE.,f16968bd-bc35-4458-841f-9b1a330799fa,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:21.637621,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -19794,7 +19794,7 @@ function will_it_fly(q, w){ } return balanced && sum <= w; }",117.92630000127247,,Completions generated successfully. Ready for display in your IDE.,6fe0774a-405b-4b4c-9cb6-33d69c775277,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... -",95,0.911 +",95,0.911 2025-09-22T13:58:21.889178,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -19835,7 +19835,7 @@ def move_one_ball(arr: List[int]) -> bool: return True ",251.3390999956755,,Completions generated successfully. Ready for display in your IDE.,6b39fc3c-f727-4a5f-bb1f-c8a1fc90fa59,1,deepseek-ai/deepseek-coder-1.3b-base," if (flag and arr[i-1] > arr[i]): -",225,0.9964 +",225,0.9964 2025-09-22T13:58:22.112650,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -19912,7 +19912,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",222.87879999930738,,Completions generated successfully. Ready for display in your IDE.,72337864-14f5-4373-a254-499edf3f0a8d,1,deepseek-ai/deepseek-coder-1.3b-base," b_tmp = b.replace(',', '.') -",192,1.0 +",192,1.0 2025-09-22T13:58:22.235247,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -19952,7 +19952,7 @@ function match_parens(lst){ } return 'No'; }",122.80230000033043,,Completions generated successfully. Ready for display in your IDE.,1d3942a2-fcf4-4f8d-b810-25c45daa1a0a,1,deepseek-ai/deepseek-coder-1.3b-base," const open = '('; -",99,1.0 +",99,1.0 2025-09-22T13:58:22.334358,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20030,7 +20030,7 @@ class Problem { } }",98.46310000284575,,Completions generated successfully. Ready for display in your IDE.,5965340b-0319-4a5a-829a-7dc0a90d1c47,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:22.439919,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -20068,7 +20068,7 @@ function is_prime(n){ } ",105.66059999837307,,Completions generated successfully. Ready for display in your IDE.,7a7f6ecd-672a-48e9-b6a2-146259e6a76f,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:58:22.543496,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20104,7 +20104,7 @@ class Problem { } }",102.64590000588214,,Completions generated successfully. Ready for display in your IDE.,126ee1ce-7e94-40f3-925c-9e81cd564eca,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:22.699626,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20143,7 +20143,7 @@ class Problem { } }",155.7228999954532,,Completions generated successfully. Ready for display in your IDE.,f18552fe-f709-409b-9922-91f13c359ccb,1,deepseek-ai/deepseek-coder-1.3b-base," minSum = currSum; -",131,1.0 +",131,1.0 2025-09-22T13:58:22.903572,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20176,7 +20176,7 @@ class Problem { } }",204.19479999691248,,Completions generated successfully. Ready for display in your IDE.,542ddbbf-e064-4710-95b9-60a23a0bfb78,1,deepseek-ai/deepseek-coder-1.3b-base," char[] chars = word.toCharArray(); -",165,1.0 +",165,1.0 2025-09-22T13:58:22.998930,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20210,7 +20210,7 @@ class Problem { } }",94.83680000266759,,Completions generated successfully. Ready for display in your IDE.,21bd12b0-c38c-473f-8aed-23ed82285a07,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:23.166971,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20239,7 +20239,7 @@ class Problem { }"," } }",168.12940000090748,,Completions generated successfully. Ready for display in your IDE.,1d052d59-329c-4ab8-b686-44da17877bea,1,deepseek-ai/deepseek-coder-1.3b-base," return Collections.max(primes); -",144,0.993 +",144,0.993 2025-09-22T13:58:23.519312,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -20263,7 +20263,7 @@ function fizz_buzz(n){ // your code here... for (var i=0 ; i bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -20288,7 +20288,7 @@ function fizz_buzz(n){ ",238.88119999901392,,Completions generated successfully. Ready for display in your IDE.,9db91061-8ee8-40de-886f-7923295e58e2,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, (n // 2) + 1): -",217,0.9969 +",217,0.9969 2025-09-22T13:58:23.884939,[1],"def fib4(n: int) -> int: """"""The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fib4(0) -> 0 @@ -20315,7 +20315,7 @@ function fizz_buzz(n){ ",126.42849999974715,,Completions generated successfully. Ready for display in your IDE.,f45e1928-f138-4063-8102-26db0d806731,1,deepseek-ai/deepseek-coder-1.3b-base," elif n == 3: -",104,1.0 +",104,1.0 2025-09-22T13:58:24.056430,[1],"//Complete the function that takes two integers and returns // the product of their unit digits. // Assume the input is always valid. @@ -20333,7 +20333,7 @@ function multiply(a, b){ } ",171.17619999771705,,Completions generated successfully. Ready for display in your IDE.,9a98e25c-6b06-4ce0-a6bb-4f36f7597b11,1,deepseek-ai/deepseek-coder-1.3b-base,"; let bStr = b.toString(); -",150,0.9659 +",150,0.9659 2025-09-22T13:58:24.168829,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -20359,7 +20359,7 @@ function correct_bracketing(brackets){ } ",112.18920000101207,,Completions generated successfully. Ready for display in your IDE.,9f11a16e-c716-42e2-be7f-1e5a7d76151a,1,deepseek-ai/deepseek-coder-1.3b-base," opens--; -",94,1.0 +",94,1.0 2025-09-22T13:58:24.367519,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -20388,7 +20388,7 @@ function f(n){ return result; }",198.35869999951683,,Completions generated successfully. Ready for display in your IDE.,31b9459e-78f7-4cd6-bcb8-ccfaff96f677,1,deepseek-ai/deepseek-coder-1.3b-base," if((i % 2) === 0 ) { -",175,0.9632 +",175,0.9632 2025-09-22T13:58:24.518082,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -20456,7 +20456,7 @@ function do_algebra(operator, operand){ } ",150.50300000439165,,Completions generated successfully. Ready for display in your IDE.,2ec4a7f5-1cf5-42cd-947a-c0f0a9c1c0f3,1,deepseek-ai/deepseek-coder-1.3b-base," do_again = true; -",123,1.0 +",123,1.0 2025-09-22T13:58:24.633565,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -20488,7 +20488,7 @@ function correct_bracketing(brackets){ } }",114.76190000394126,,Completions generated successfully. Ready for display in your IDE.,c6eef92e-6a39-4ddc-9332-5416cc0c430a,1,deepseek-ai/deepseek-coder-1.3b-base," opens.pop(); -",92,1.0 +",92,1.0 2025-09-22T13:58:24.686506,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -20509,7 +20509,7 @@ function correct_bracketing(brackets){ else: return n * special_factorial(n-1) * factorial(n-1) -",52.60239999915939,,Completions generated successfully. Ready for display in your IDE.,c503a4e3-1c00-4493-97ad-2b5b0f1ec356,1,deepseek-ai/deepseek-coder-1.3b-base,,31,1.0 +",52.60239999915939,,Completions generated successfully. Ready for display in your IDE.,c503a4e3-1c00-4493-97ad-2b5b0f1ec356,1,deepseek-ai/deepseek-coder-1.3b-base,,31,1.0 2025-09-22T13:58:24.864895,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20558,7 +20558,7 @@ class Problem { } }",178.84629999753088,,Completions generated successfully. Ready for display in your IDE.,e26d296e-13bf-4ab4-9b98-743100cda4e5,1,deepseek-ai/deepseek-coder-1.3b-base," String[] ss = test.split("" ""); -",152,1.0 +",152,1.0 2025-09-22T13:58:25.133062,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20594,7 +20594,7 @@ class Problem { } }",267.8186999983154,,Completions generated successfully. Ready for display in your IDE.,936efa52-f75d-43dd-9d5b-4ba64a4930b9,1,deepseek-ai/deepseek-coder-1.3b-base," if (lst1.get(i) % 2 != 0) { -",246,1.0 +",246,1.0 2025-09-22T13:58:25.199434,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -20632,7 +20632,7 @@ function exchange(lst1, lst2){ } } ",66.26149999647168,,Completions generated successfully. Ready for display in your IDE.,a70278c8-a7ee-4b4e-9544-9cfc852c59c0,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,1.0 + ",48,1.0 2025-09-22T13:58:25.384729,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20672,7 +20672,7 @@ class Problem { return arrayList; } }",185.02869999792892,,Completions generated successfully. Ready for display in your IDE.,3028eb77-4998-4bdf-9773-c1470dcea6fc,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList arrayList = new ArrayList<>(); -",162,1.0 +",162,1.0 2025-09-22T13:58:25.464602,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -20690,7 +20690,7 @@ function x_or_y(n, x, y){ }"," return x; }",79.35390000056941,,Completions generated successfully. Ready for display in your IDE.,6ba55fcb-ee82-442f-a9db-a80ff79e37d6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:58:25.558672,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -20718,7 +20718,7 @@ function x_or_y(n, x, y){ ",94.10689999640454,,Completions generated successfully. Ready for display in your IDE.,e47ecbea-acb5-4c35-a59a-24503f2a2be0,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T13:58:25.716930,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -20776,7 +20776,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.1506000002264,,Completions generated successfully. Ready for display in your IDE.,6f0b0538-8467-4e23-8ce0-e3081fbd24bf,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('D-') -",131,1.0 +",131,1.0 2025-09-22T13:58:25.883326,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -20806,7 +20806,7 @@ function pairs_sum_to_zero(l){ } else { break } -",143,0.9374 +",143,0.9374 2025-09-22T13:58:26.026148,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -20829,7 +20829,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",142.15079999848967,,Completions generated successfully. Ready for display in your IDE.,ff8b0756-3d95-4e60-864f-c582d8768dab,1,deepseek-ai/deepseek-coder-1.3b-base," digit = int(ch) -",119,1.0 +",119,1.0 2025-09-22T13:58:26.224127,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20868,7 +20868,7 @@ class Problem { } else { return false; } -",171,1.0 +",171,1.0 2025-09-22T13:58:26.439019,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20898,7 +20898,7 @@ class Problem { }"," } }",214.6400999990874,,Completions generated successfully. Ready for display in your IDE.,6f988f2c-6f31-4b19-97c5-686bbdf35eea,1,deepseek-ai/deepseek-coder-1.3b-base," return newString.toString(); -",114,0.9642 +",114,0.9642 2025-09-22T13:58:26.605572,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -20936,7 +20936,7 @@ function even_odd_palindrome(n){ } }",},166.84829999576323,,Completions generated successfully. Ready for display in your IDE.,9143ede2-8e0b-4bb8-9c09-1ce62d2c6a03,1,deepseek-ai/deepseek-coder-1.3b-base," console.log([even, odd]); -",141,1.0 +",141,1.0 2025-09-22T13:58:26.807051,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -20975,7 +20975,7 @@ function find_max(words){ return max_unique_word; }",200.90049999998882,,Completions generated successfully. Ready for display in your IDE.,193381ee-832a-48ce-a8e5-2258f99eb81f,1,deepseek-ai/deepseek-coder-1.3b-base," if (word <= max_unique_word) { -",172,0.9731 +",172,0.9731 2025-09-22T13:58:26.894595,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20992,7 +20992,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { return (new ArrayList(l.stream().filter(x -> x > 0).collect(Collectors.toList())));",},87.36759999737842,,Completions generated successfully. Ready for display in your IDE.,cc4a10e5-d589-4463-9639-6460cfc43ea0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:26.963870,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21024,7 +21024,7 @@ class Problem { } return isPalindrome; }",,68.94160000229022,,Completions generated successfully. Ready for display in your IDE.,2822b135-30bd-4fbc-a54d-c008ebb6e289,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,0.9511 +}",45,0.9511 2025-09-22T13:58:27.289353,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -21049,7 +21049,7 @@ function compare(game, guess){ ",325.1814999966882,,Completions generated successfully. Ready for display in your IDE.,90600093-62c2-4541-b798-97da3edb92a1,1,deepseek-ai/deepseek-coder-1.3b-base," const diff = Math.abs((guess[i]-game[i])) result.push(diff) -",300,0.9945 +",300,0.9945 2025-09-22T13:58:27.364604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21096,7 +21096,7 @@ class Problem { return l_s; } }",75.6143000035081,,Completions generated successfully. Ready for display in your IDE.,13d0cfe3-af84-4821-aa71-2ff5a089c043,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,0.9367 +",46,0.9367 2025-09-22T13:58:27.432545,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21124,7 +21124,7 @@ class Problem { } return boredoms; }",,67.17690000368748,,Completions generated successfully. Ready for display in your IDE.,2395cfb6-fb0b-4a10-91b0-b1acf02f44a2,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:58:27.681496,[1],"//Task // Write a function that takes a string as input and returns the sum of the upper characters only' // ASCII codes. @@ -21149,7 +21149,7 @@ function digitSum(s){ return sum; }",249.35069999628467,,Completions generated successfully. Ready for display in your IDE.,61d8f1b3-5d9f-41e9-8718-9439329f5081,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i < s.length; ++i) { -",226,0.9532 +",226,0.9532 2025-09-22T13:58:28.106834,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -21170,7 +21170,7 @@ function digitSum(s){ ",424.5112999997218,,Completions generated successfully. Ready for display in your IDE.,9420f713-d86a-43b9-a243-ee673ce863ce,1,deepseek-ai/deepseek-coder-1.3b-base," if not isinstance(n, int): raise TypeError(""modp: n must be an integer"") if not isinstance(p, int): -",402,0.9741 +",402,0.9741 2025-09-22T13:58:28.254430,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -21200,7 +21200,7 @@ function closest_integer(value){ } ",148.05069999420084,,Completions generated successfully. Ready for display in your IDE.,f5626f26-91d5-4cc9-ad85-ae1f0e358323,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.floor(value); -",123,1.0 +",123,1.0 2025-09-22T13:58:28.361697,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -21226,7 +21226,7 @@ function is_multiply_prime(a){ } }",106.46429999906104,,Completions generated successfully. Ready for display in your IDE.,ed9a4dd0-17a5-47a4-87dd-d5e19700119f,1,deepseek-ai/deepseek-coder-1.3b-base," i++; -",83,1.0 +",83,1.0 2025-09-22T13:58:28.488965,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -21270,7 +21270,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",127.49160000385018,,Completions generated successfully. Ready for display in your IDE.,775f966f-94db-4fa4-b9bc-4071ee650e79,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",100,1.0 +",100,1.0 2025-09-22T13:58:28.587756,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -21301,7 +21301,7 @@ function skjkasdkd(lst){ sum += Number(digits[i]);"," return sum; }",98.61800000362564,,Completions generated successfully. Ready for display in your IDE.,5aa019d9-6e68-4817-a943-aa5adee4a5b4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",75,1.0 +",75,1.0 2025-09-22T13:58:28.744750,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21337,7 +21337,7 @@ class Problem { }",156.32049999840092,,Completions generated successfully. Ready for display in your IDE.,1f0f29cf-8aff-473a-b196-ea7cf7217165,1,deepseek-ai/deepseek-coder-1.3b-base," long i = 5; -",132,1.0 +",132,1.0 2025-09-22T13:58:28.831304,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -21368,7 +21368,7 @@ function same_chars(s0, s1){ ",86.84019999782322,,Completions generated successfully. Ready for display in your IDE.,167ba9fb-7017-4772-b384-03e902eb47af,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:28.948524,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -21396,7 +21396,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return False return True ",116.42969999957131,,Completions generated successfully. Ready for display in your IDE.,25cb8cf0-f93c-40e8-8d1b-0d3187b9df6f,1,deepseek-ai/deepseek-coder-1.3b-base," if not dict: -",84,1.0 +",84,1.0 2025-09-22T13:58:29.060232,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -21468,7 +21468,7 @@ function minPath(grid, k){ return path; }",111.86769999767421,,Completions generated successfully. Ready for display in your IDE.,b8066397-188d-4a81-985e-ff6dd7e13535,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:58:29.323687,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -21518,7 +21518,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",263.15559999784455,,Completions generated successfully. Ready for display in your IDE.,ca8ba7b8-dce9-4e48-a489-181643c00925,1,deepseek-ai/deepseek-coder-1.3b-base," if(number % i == 0 || number % (i + 2) == 0){ -",239,0.9693 +",239,0.9693 2025-09-22T13:58:29.391577,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21549,7 +21549,7 @@ class Problem { } return result; }",,67.28319999820087,,Completions generated successfully. Ready for display in your IDE.,7205709c-0058-4534-8d7a-5fc8dadd83ef,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:29.692673,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -21589,7 +21589,7 @@ function is_prime(n){ ",300.86080000182847,,Completions generated successfully. Ready for display in your IDE.,60568158-4630-4aa6-b375-f352b8b1a865,1,deepseek-ai/deepseek-coder-1.3b-base," //if n < 2 then it's not prime if (n<2) { -",278,0.9594 +",278,0.9594 2025-09-22T13:58:29.753416,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -21607,7 +21607,7 @@ def sort_even(l: List[int]) -> List[int]: temp.append(l[i])"," for i in range(0, len(l), 2): l[i] = temp[i // 2] return l",60.85200000234181,,Completions generated successfully. Ready for display in your IDE.,6b59ec6b-28b5-4374-9a73-a7e935c13ff2,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:29.922325,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21648,7 +21648,7 @@ class Problem { } }",168.23430000658846,,Completions generated successfully. Ready for display in your IDE.,1a42a36f-48dd-49e7-aaf0-c6ac8971ce51,1,deepseek-ai/deepseek-coder-1.3b-base," for (Long i : good) { -",143,1.0 +",143,1.0 2025-09-22T13:58:30.060573,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -21725,7 +21725,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",138.3261000009952,,Completions generated successfully. Ready for display in your IDE.,148aedf7-dd57-4819-a8f1-ef3140b4e813,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",88,1.0 +",88,1.0 2025-09-22T13:58:30.156142,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -21769,7 +21769,7 @@ function bf(planet1, planet2){ return res; }",94.94750000158092,,Completions generated successfully. Ready for display in your IDE.,66b4615f-d1be-4070-b3f7-30e2febca28c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:30.223268,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21800,7 +21800,7 @@ class Problem { } return inc || dec; }",,67.14269999793032,,Completions generated successfully. Ready for display in your IDE.,08c20b9e-4f4a-4fc2-a0ea-ad19c7986681,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:58:30.331711,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -21833,7 +21833,7 @@ class Problem { ",108.17540000425652,,Completions generated successfully. Ready for display in your IDE.,1f01371e-87f6-4f99-bcb2-ea9266dd337e,1,deepseek-ai/deepseek-coder-1.3b-base," j = 1 -",84,1.0 +",84,1.0 2025-09-22T13:58:30.511377,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -21861,7 +21861,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: return False ",179.77790000441018,,Completions generated successfully. Ready for display in your IDE.,edbf4995-df29-45eb-95fe-7476eb5851bc,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): -",155,1.0 +",155,1.0 2025-09-22T13:58:30.606811,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21900,7 +21900,7 @@ class Problem { } return res;",},95.27630000229692,,Completions generated successfully. Ready for display in your IDE.,842fd954-70e6-4de3-bd1b-fb45ee8607ab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:30.696685,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21930,7 +21930,7 @@ class Problem { } }",89.84639999835053,,Completions generated successfully. Ready for display in your IDE.,9246213f-ac5e-472f-9d13-7c7520a1f962,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T13:58:30.836555,[1],"from typing import List def max_fill(grid: List[List[int]], capacity: int) -> int: @@ -21967,7 +21967,7 @@ def max_fill(grid: List[List[int]], capacity: int) -> int: return times ",139.23119999526534,,Completions generated successfully. Ready for display in your IDE.,174d2a44-9ba9-4c91-8f90-108006de02ed,1,deepseek-ai/deepseek-coder-1.3b-base," for row in grid: -",114,1.0 +",114,1.0 2025-09-22T13:58:31.025090,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -22007,7 +22007,7 @@ function pluck(arr){ }else{ continue; } -",161,0.9825 +",161,0.9825 2025-09-22T13:58:31.104330,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -22039,7 +22039,7 @@ function pluck(arr){ return count ",79.1828999936115,,Completions generated successfully. Ready for display in your IDE.,6c0b4034-ddd6-4464-b8f6-31febd9a137d,1,deepseek-ai/deepseek-coder-1.3b-base," - ",43,1.0 + ",43,1.0 2025-09-22T13:58:31.215915,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -22074,7 +22074,7 @@ function search(lst){ }","} ",111.10719999851426,,Completions generated successfully. Ready for display in your IDE.,f16d3fbe-7314-445e-8dc5-a7f390752c9a,1,deepseek-ai/deepseek-coder-1.3b-base," return max; -",87,0.8919 +",87,0.8919 2025-09-22T13:58:31.330243,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22112,7 +22112,7 @@ class Problem { } }",114.8107999979402,,Completions generated successfully. Ready for display in your IDE.,f95e4171-cbf3-41f8-94aa-28241c1cb435,1,deepseek-ai/deepseek-coder-1.3b-base," max--; -",89,1.0 +",89,1.0 2025-09-22T13:58:31.417725,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22135,7 +22135,7 @@ class Problem { .filter(i -> s.charAt(i) == 'A' || s.charAt(i) == 'E' || s.charAt(i) == 'I' || s.charAt(i) == 'O' || s.charAt(i) == 'U') .count();",},86.75750000111293,,Completions generated successfully. Ready for display in your IDE.,28e8da89-518c-461e-8a25-079c407b7d79,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:31.502456,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -22165,7 +22165,7 @@ function words_in_sentence(sentence){ }).join(' '); }",84.92660000047181,,Completions generated successfully. Ready for display in your IDE.,a60f7fca-d87f-451d-b2e8-cad3af97ba20,1,deepseek-ai/deepseek-coder-1.3b-base," }; -",61,0.9555 +",61,0.9555 2025-09-22T13:58:31.687923,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -22206,7 +22206,7 @@ def maximum(arr: List[int], k: int) -> List[int]: arr.remove(max_num) return [max_num] + get_max(arr, k - 1) ",,185.44029999611666,,Completions generated successfully. Ready for display in your IDE.,d8cb5692-4f16-4e7b-971e-bb6cdb2854a7,1,deepseek-ai/deepseek-coder-1.3b-base," - return get_max(arr, k)",161,0.9776 + return get_max(arr, k)",161,0.9776 2025-09-22T13:58:31.845833,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -22222,7 +22222,7 @@ function count_distinct_characters(string){ } ",157.214799997746,,Completions generated successfully. Ready for display in your IDE.,416c93c7-3efe-4eb3-ac1f-a55c263eef49,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(arr[i]); -",134,1.0 +",134,1.0 2025-09-22T13:58:32.214304,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -22262,7 +22262,7 @@ function valid_date(date){ } ",368.67509999865433,,Completions generated successfully. Ready for display in your IDE.,97d3ef58-fa08-4ffa-933c-188007bad191,1,deepseek-ai/deepseek-coder-1.3b-base," //split by - then convert each element into an integer using parseint() let date_arr = date.split('-'); -",338,0.8831 +",338,0.8831 2025-09-22T13:58:32.393449,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -22277,7 +22277,7 @@ function filter_integers(values){ } ",178.93920000642538,,Completions generated successfully. Ready for display in your IDE.,e1716827-bbfe-46c5-a861-3081efa479c0,1,deepseek-ai/deepseek-coder-1.3b-base," _.each(values, function (value) { -",156,1.0 +",156,1.0 2025-09-22T13:58:32.480699,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -22309,7 +22309,7 @@ function correct_bracketing(brackets){ } }",86.58100000320701,,Completions generated successfully. Ready for display in your IDE.,34b9918d-6709-4641-9b5b-b0054ef8b525,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:32.702521,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22338,7 +22338,7 @@ class Problem { } }",221.86669999791775,,Completions generated successfully. Ready for display in your IDE.,7aa5a75b-dcad-414f-a5ff-7e7f0d87fa4f,1,deepseek-ai/deepseek-coder-1.3b-base," long oranges = Long.parseLong(splitted[1]); -",197,1.0 +",197,1.0 2025-09-22T13:58:32.921044,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -22363,7 +22363,7 @@ class Problem { elif n == 2: return 3 else: -",201,0.9824 +",201,0.9824 2025-09-22T13:58:33.067302,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -22399,7 +22399,7 @@ class Problem { ",146.05270000174642,,Completions generated successfully. Ready for display in your IDE.,14ef690a-b8d7-452d-a45a-cf43c1197228,1,deepseek-ai/deepseek-coder-1.3b-base," 900: 'cm', -",125,1.0 +",125,1.0 2025-09-22T13:58:33.356600,[1],"//Write a function that takes an integer a and returns true // if this ingeger is a cube of some integer number. // Note: you may assume the input is always valid. @@ -22422,7 +22422,7 @@ function iscube(a){ return false; } }",289.21429999900283,,Completions generated successfully. Ready for display in your IDE.,a0535d34-6281-42f4-a130-c47f4bd2c386,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.pow((int)(Math.sqrt(a)),3) == Math.abs(a)){ -",268,0.9799 +",268,0.9799 2025-09-22T13:58:33.516942,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22467,7 +22467,7 @@ class Problem { } }",159.54010000132257,,Completions generated successfully. Ready for display in your IDE.,dfb4bbf9-f107-4cd3-a4a1-ac21261fb886,1,deepseek-ai/deepseek-coder-1.3b-base," if (sum <= w) { -",133,1.0 +",133,1.0 2025-09-22T13:58:33.687838,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22493,7 +22493,7 @@ class Problem { return levels; } }",170.92119999870192,,Completions generated successfully. Ready for display in your IDE.,12ff86b2-5ca7-4976-a7cc-e1a0a101f45f,1,deepseek-ai/deepseek-coder-1.3b-base," List levels = new ArrayList<>(); -",146,1.0 +",146,1.0 2025-09-22T13:58:34.394030,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -22530,7 +22530,7 @@ print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",705.6670000019949, temp_index = i temp2_index = j elif abs(numbers[i] - numbers[j]) > temp: - temp = abs(numbers[i] - numbers[j",678,0.9896 + temp = abs(numbers[i] - numbers[j",678,0.9896 2025-09-22T13:58:34.492856,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22574,7 +22574,7 @@ class Problem { } }",98.76919999805978,,Completions generated successfully. Ready for display in your IDE.,421faf44-362a-457a-8ae1-158d4e35a010,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:34.802793,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22607,7 +22607,7 @@ class Problem { }",310.1033999992069,,Completions generated successfully. Ready for display in your IDE.,050babb3-d124-451f-a963-53d1c66eec6d,1,deepseek-ai/deepseek-coder-1.3b-base," long value = hashmap.get(key).longValue(); if (value == 1) { -",284,0.9971 +",284,0.9971 2025-09-22T13:58:35.065197,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -22684,7 +22684,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",261.9316999989678,,Completions generated successfully. Ready for display in your IDE.,779019bb-9d33-4ffd-a411-48a373a2b9ef,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, float) and isinstance(b, int): -",233,1.0 +",233,1.0 2025-09-22T13:58:35.160306,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22717,7 +22717,7 @@ class Problem { } return sum;",},94.53619999840157,,Completions generated successfully. Ready for display in your IDE.,ee832366-b832-4f79-acb0-ba0ca943ba06,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:35.223912,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -22755,7 +22755,7 @@ function find_max(words){ } return max_unique_word; }",64.08689999807393,,Completions generated successfully. Ready for display in your IDE.,a6c1a49f-36f3-4259-a63b-d44c10f65a97,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:35.474065,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -22776,7 +22776,7 @@ function longest(strings){ } ",249.7019000002183,,Completions generated successfully. Ready for display in your IDE.,a03a8dd2-1bf8-4f2c-bb45-52a2303b022e,1,deepseek-ai/deepseek-coder-1.3b-base," if ((result === null || result.length <= strings[i].length)) { -",226,0.9655 +",226,0.9655 2025-09-22T13:58:35.542246,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -22813,7 +22813,7 @@ function is_prime(n){ return true; } ",68.03909999871394,,Completions generated successfully. Ready for display in your IDE.,b709c03e-2db4-4b4c-a247-e96a15ce365b,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:35.785884,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -22835,7 +22835,7 @@ function find_closest_elements(numbers){ } ",242.73469999752706,,Completions generated successfully. Ready for display in your IDE.,8537bf36-3bd5-4d53-8ef8-d0d940160c03,1,deepseek-ai/deepseek-coder-1.3b-base," closest = Math.abs(numbers[i] - numbers[j]); -",217,0.9856 +",217,0.9856 2025-09-22T13:58:35.907251,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22863,7 +22863,7 @@ class Problem { return count; } }",121.4442999989842,,Completions generated successfully. Ready for display in your IDE.,90a52062-a906-4295-8b90-f78552905b48,1,deepseek-ai/deepseek-coder-1.3b-base," long count = 0; -",94,1.0 +",94,1.0 2025-09-22T13:58:36.102837,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22931,7 +22931,7 @@ class Problem { } }",195.51249999494758,,Completions generated successfully. Ready for display in your IDE.,c857ed9b-356e-47d1-b790-a44b33320836,1,deepseek-ai/deepseek-coder-1.3b-base," long previous_number = 0L; -",157,0.9905 +",157,0.9905 2025-09-22T13:58:36.184739,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22998,7 +22998,7 @@ class Problem { } } }",81.86569999816129,,Completions generated successfully. Ready for display in your IDE.,91bb740a-707c-43ef-9000-7462596a055e,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:58:36.294022,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -23029,7 +23029,7 @@ function same_chars(s0, s1){ ",108.5562000007485,,Completions generated successfully. Ready for display in your IDE.,dfc6360f-215f-4f35-aa9a-0b4ada191a9f,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",84,1.0 +",84,1.0 2025-09-22T13:58:36.392482,[1],"def circular_shift(x: int, shift: int) -> str: """"""Circular shift the digits of the integer x, shift the digits right by shift and return the result as a string. @@ -23045,7 +23045,7 @@ function same_chars(s0, s1){ ",98.77099999721395,,Completions generated successfully. Ready for display in your IDE.,bcf332af-bf68-42de-8693-67c1dbdece38,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T13:58:36.556610,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23123,7 +23123,7 @@ class Problem { } }",163.74070000165375,,Completions generated successfully. Ready for display in your IDE.,d569ed94-6ae5-4807-bd2c-9a0ce6c18aac,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 100; -",136,1.0 +",136,1.0 2025-09-22T13:58:36.975073,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -23156,7 +23156,7 @@ function count_up_to(n){ // check all odd nums up to sqrt(n), then add them as primes until we reach n let output = new Array(); -",393,0.8927 +",393,0.8927 2025-09-22T13:58:37.081041,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23179,7 +23179,7 @@ class Problem { } }",106.44219999812776,,Completions generated successfully. Ready for display in your IDE.,6a359dc6-32d2-4d57-bd0b-b706b9a2c9f7,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",83,1.0 +",83,1.0 2025-09-22T13:58:37.176537,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -23206,7 +23206,7 @@ function strange_sort_list(lst){ return output; }",95.44420000020182,,Completions generated successfully. Ready for display in your IDE.,1ddc9e9d-2423-4f82-a574-e6d3114556b4,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",72,1.0 +",72,1.0 2025-09-22T13:58:37.460143,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23251,7 +23251,7 @@ class Problem { } }",282.94170000299346,,Completions generated successfully. Ready for display in your IDE.,4e1490a9-551d-4754-ba74-0982a5ee6302,1,deepseek-ai/deepseek-coder-1.3b-base," List words = Arrays.asList(sentence.split("" "")); -",257,0.9752 +",257,0.9752 2025-09-22T13:58:37.555061,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -23288,7 +23288,7 @@ function max_fill(grid, capacity){ } ",94.69009999884292,,Completions generated successfully. Ready for display in your IDE.,fc5bc54d-ada1-434c-bf0c-c748dd6212eb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,0.9223 +",68,0.9223 2025-09-22T13:58:37.671917,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -23326,7 +23326,7 @@ function is_prime(n){ ","} ",116.92219999531517,,Completions generated successfully. Ready for display in your IDE.,fc31c80f-caf0-4edd-a540-043de4639ffd,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",94,1.0 +",94,1.0 2025-09-22T13:58:37.787494,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23364,7 +23364,7 @@ class Problem { } }",114.67350000020815,,Completions generated successfully. Ready for display in your IDE.,d63e4b9b-4ef0-4748-989e-b7cce8f6c36f,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",89,1.0 +",89,1.0 2025-09-22T13:58:37.990263,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -23391,7 +23391,7 @@ class Problem { return ch return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",202.98420000472106,,Completions generated successfully. Ready for display in your IDE.,9d1b3084-43d0-4bd0-90a8-f47b7e2465e2,1,deepseek-ai/deepseek-coder-1.3b-base," def swap_case(ch: str) -> str: -",176,0.9947 +",176,0.9947 2025-09-22T13:58:38.077803,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -23423,7 +23423,7 @@ function correct_bracketing(brackets){ } }",86.8244000012055,,Completions generated successfully. Ready for display in your IDE.,bb79e5c1-85be-4e15-86ff-bdef7dd4de77,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:38.783590,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23471,7 +23471,7 @@ class Problem { for (int i = 2; i < 50; ++i) { int count = 0; for (int j = 2; j < i; ++j) { - if (i % j == 0",679,0.9787 + if (i % j == 0",679,0.9787 2025-09-22T13:58:38.870659,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -23493,7 +23493,7 @@ function find_closest_elements(numbers){ } ",87.45059999637306,,Completions generated successfully. Ready for display in your IDE.,2ae2bb2c-c7e8-43fa-a458-f5eb168fea23,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:39.040838,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -23515,7 +23515,7 @@ function cycpattern_check(a, b){ }","} ",169.2934000020614,,Completions generated successfully. Ready for display in your IDE.,48d7941d-ab91-48ed-ab87-fc794eb8bf6f,1,deepseek-ai/deepseek-coder-1.3b-base," console.log('result: ', result); -",146,0.9349 +",146,0.9349 2025-09-22T13:58:39.126144,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -23534,7 +23534,7 @@ function sum_product(numbers){ } ",84.8236999954679,,Completions generated successfully. Ready for display in your IDE.,90f3b29f-2bc5-4d5a-b9bc-b24740a145bb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:39.266410,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -23563,7 +23563,7 @@ function f(n){ return result; }",140.6563999989885,,Completions generated successfully. Ready for display in your IDE.,88484d99-ff2b-4b4e-8608-f5dd1cf70347,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(sum); -",117,1.0 +",117,1.0 2025-09-22T13:58:39.359864,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -23606,7 +23606,7 @@ function get_odd_collatz(n){ } ",93.51819999574218,,Completions generated successfully. Ready for display in your IDE.,803d5a50-04de-4341-a843-a5202a3a9232,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:40.061202,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23644,7 +23644,7 @@ class Problem { while (!found && j < 5) { if ((vowels[j] == arr[i]) || (vowels[j+1] == arr[i])){ found = true; - ",671,0.9603 + ",671,0.9603 2025-09-22T13:58:40.137106,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23693,7 +23693,7 @@ class Problem { } return s; }",,75.90390000405023,,Completions generated successfully. Ready for display in your IDE.,c8980740-7ff0-42a0-bb06-908a9f7159ec,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,1.0 +}",50,1.0 2025-09-22T13:58:40.549208,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23755,7 +23755,7 @@ class Problem { } }",411.84620000422,,Completions generated successfully. Ready for display in your IDE.,20d623fb-b549-4d16-aba6-c76de911367b,1,deepseek-ai/deepseek-coder-1.3b-base," if ((Math.sqrt(number)).equals(Math.floor(Math.sqrt(number))) && !isPrime(number)) { -",383,0.904 +",383,0.904 2025-09-22T13:58:40.720598,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -23783,7 +23783,7 @@ class Problem { ",170.89589999523014,,Completions generated successfully. Ready for display in your IDE.,356f8c6d-4cb7-446c-88d7-e9392346b7f2,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(n - 2): -",148,0.9495 +",148,0.9495 2025-09-22T13:58:40.814126,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -23822,7 +23822,7 @@ function find_max(words){ return max_unique_word; }",93.29659999639262,,Completions generated successfully. Ready for display in your IDE.,7b295ec8-0282-45cc-bd37-33ef96f92762,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:40.909176,[1],"//Return median of elements in the array l. // >>> median([3, 1, 2, 4, 5]) // 3 @@ -23837,7 +23837,7 @@ function median(l){ } ",95.41280000121333,,Completions generated successfully. Ready for display in your IDE.,39c4c0e0-0169-4c87-9a9d-b7a9e7b36c45,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",74,0.959 +",74,0.959 2025-09-22T13:58:41.276075,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23863,7 +23863,7 @@ class Problem { } }",366.31969999871217,,Completions generated successfully. Ready for display in your IDE.,35411230-734e-40aa-a5b1-b57ad06081cd,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = Math.max(x,y); i >= Math.min(x,y)-1; --i) { -",343,0.9936 +",343,0.9936 2025-09-22T13:58:41.335579,[1],"//Your task is to implement a function that will simplify the expression // x * n. The function returns true if x * n evaluates to a whole number and false // otherwise. Both x and n, are string representation of a fraction, and have the following format, @@ -23882,7 +23882,7 @@ function simplify(x, n){ let nDenominator = parseInt(n.split('/')[1]); let xTimesN = xNumerator * nNumerator / (xDenominator * nDenominator); return Number.isInteger(xTimesN);",,59.478800001670606,,Completions generated successfully. Ready for display in your IDE.,80bf699e-7903-4fe2-b620-1750674fa9a4,1,deepseek-ai/deepseek-coder-1.3b-base," -}",37,1.0 +}",37,1.0 2025-09-22T13:58:41.465105,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23915,7 +23915,7 @@ class Problem { } }",128.91279999894323,,Completions generated successfully. Ready for display in your IDE.,ad67021d-0cf9-45fb-9bba-a36d6762e278,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append("" ""); -",104,1.0 +",104,1.0 2025-09-22T13:58:41.610769,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -23965,7 +23965,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",145.69399999891175,,Completions generated successfully. Ready for display in your IDE.,b563bc84-e45d-4617-96ee-3b50b16245c6,1,deepseek-ai/deepseek-coder-1.3b-base," a = a + b; -",119,0.9934 +",119,0.9934 2025-09-22T13:58:41.694165,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -23979,7 +23979,7 @@ function largest_divisor(n){ }"," return ld; }",83.4125999972457,,Completions generated successfully. Ready for display in your IDE.,f6fcfa81-feb5-48d2-a6ca-e06e0e3ce1b4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:58:41.804159,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24018,7 +24018,7 @@ class Problem { } }",109.2516999997315,,Completions generated successfully. Ready for display in your IDE.,5523be2a-7ca3-4c67-968f-4fcfe313faed,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:41.892373,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -24041,7 +24041,7 @@ function encode(message){ return result; }",88.74210000067251,,Completions generated successfully. Ready for display in your IDE.,9f2f2586-eb00-49eb-836e-41475bc40ce5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:42.020173,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24081,7 +24081,7 @@ class Problem { } }",126.93250000302214,,Completions generated successfully. Ready for display in your IDE.,f54b5f22-dae5-436d-9e93-2fef967ee662,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",100,1.0 +",100,1.0 2025-09-22T13:58:42.121787,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24117,7 +24117,7 @@ class Problem { } }",101.57299999991665,,Completions generated successfully. Ready for display in your IDE.,866cb142-4428-4aad-a2f5-3f53308cc835,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:58:42.260269,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -24159,7 +24159,7 @@ def maximum(arr: List[int], k: int) -> List[int]: return sorted(get_max(arr, k)) ",138.39899999584304,,Completions generated successfully. Ready for display in your IDE.,1408a2db-d2fa-45aa-96dc-5cac929ac901,1,deepseek-ai/deepseek-coder-1.3b-base," elif k == 1: -",113,0.9859 +",113,0.9859 2025-09-22T13:58:42.459322,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -24181,7 +24181,7 @@ function can_arrange(arr){ } ",198.53319999674568,,Completions generated successfully. Ready for display in your IDE.,b5be9715-a897-488d-be96-6c5e660f2a7a,1,deepseek-ai/deepseek-coder-1.3b-base," max = Math.max(max, arr[i]); -",175,1.0 +",175,1.0 2025-09-22T13:58:42.640046,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -24213,7 +24213,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",180.93550000048708,,Completions generated successfully. Ready for display in your IDE.,7deb464b-a205-4edf-a5f1-d9c6365bf8df,1,deepseek-ai/deepseek-coder-1.3b-base," if letters[letter] > most_count: -",156,1.0 +",156,1.0 2025-09-22T13:58:42.738150,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24252,7 +24252,7 @@ class Problem { } return minSum;",},97.80429999955231,,Completions generated successfully. Ready for display in your IDE.,15fd7d74-6cd4-4752-bc74-a97c6cf30e35,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:43.084995,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24294,7 +24294,7 @@ class Problem { } }",346.35910000361037,,Completions generated successfully. Ready for display in your IDE.,ff1a3c4c-6500-470a-905b-b04cb8b48327,1,deepseek-ai/deepseek-coder-1.3b-base," if (!lst.contains(s + ""0"") && !lst.contains(s + ""-1"")) { -",319,0.9574 +",319,0.9574 2025-09-22T13:58:43.242056,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -24329,7 +24329,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(-1) else:"," return sum(map(abs, arr)) * product(sign_arr)",157.50870000192663,,Completions generated successfully. Ready for display in your IDE.,fb2eb877-3715-4d69-8eb3-4126f7a453e1,1,deepseek-ai/deepseek-coder-1.3b-base," sign_arr.append(0) -",134,1.0 +",134,1.0 2025-09-22T13:58:43.470168,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -24397,7 +24397,7 @@ function do_algebra(operator, operand){ } ",227.25419999915175,,Completions generated successfully. Ready for display in your IDE.,c378f314-7530-4980-81d9-1c8a7a9378ae,1,deepseek-ai/deepseek-coder-1.3b-base," operand.splice(count + 1, 1); -",197,1.0 +",197,1.0 2025-09-22T13:58:43.524383,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -24419,7 +24419,7 @@ function do_algebra(operator, operand){ return True"," return False ",53.74380000284873,,Completions generated successfully. Ready for display in your IDE.,cab36592-2469-4065-b00d-ba5fb74b5059,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:43.619084,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -24445,7 +24445,7 @@ function is_multiply_prime(a){ } }",95.19970000110334,,Completions generated successfully. Ready for display in your IDE.,a0a5f016-82ab-4d4f-ada6-2f4327575f82,1,deepseek-ai/deepseek-coder-1.3b-base," -",54,1.0 +",54,1.0 2025-09-22T13:58:44.317373,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24475,7 +24475,7 @@ class Problem { /* Given an array of integers, return the sum of all the numbers in the array except the largest number. If there are multiple elements, return the sum of those two numbers. - >>> getSum",672,0.9191 + >>> getSum",672,0.9191 2025-09-22T13:58:44.543446,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -24519,7 +24519,7 @@ function compare_one(a, b){ }else { console.log('error'); } -",197,0.8892 +",197,0.8892 2025-09-22T13:58:44.639349,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -24560,7 +24560,7 @@ function int_to_mini_roman(number){ } ",95.83279999787919,,Completions generated successfully. Ready for display in your IDE.,2b83fb92-5bbc-44a5-b042-41457ad1e445,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:58:44.765604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24622,7 +24622,7 @@ class Problem { } }",126.40169999940554,,Completions generated successfully. Ready for display in your IDE.,9508b8cd-0a04-4f4e-bdd0-8162549869ee,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:58:44.999694,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24665,7 +24665,7 @@ class Problem { return false; } }",234.00190000393195,,Completions generated successfully. Ready for display in your IDE.,54419078-ca7c-479a-824f-46f99a45e1b9,1,deepseek-ai/deepseek-coder-1.3b-base," if (n < 0 || n > 99999L) { -",207,0.878 +",207,0.878 2025-09-22T13:58:45.231847,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24688,7 +24688,7 @@ class Problem { } }",231.13699999521486,,Completions generated successfully. Ready for display in your IDE.,66c31d6f-e41a-4996-ba71-190579d978d4,1,deepseek-ai/deepseek-coder-1.3b-base," answer += Character.toLowerCase(string.charAt(i)); -",207,1.0 +",207,1.0 2025-09-22T13:58:45.428705,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -24716,7 +24716,7 @@ function pairs_sum_to_zero(l){ } ",196.88010000390932,,Completions generated successfully. Ready for display in your IDE.,cb26c683-8db3-40f6-9ff4-1618626d2229,1,deepseek-ai/deepseek-coder-1.3b-base," while (i < l.length-1) { -",172,0.9443 +",172,0.9443 2025-09-22T13:58:45.542592,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24750,7 +24750,7 @@ class Problem { return open == 0; } }",114.0088999964064,,Completions generated successfully. Ready for display in your IDE.,7c8d43cf-b448-420e-9412-9ace3384c124,1,deepseek-ai/deepseek-coder-1.3b-base," int open = 0; -",91,1.0 +",91,1.0 2025-09-22T13:58:45.649716,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24794,7 +24794,7 @@ class Problem { return firstDigit % 2 != 0 && lastDigit % 2 != 0; }).count();",},106.42860000371002,,Completions generated successfully. Ready for display in your IDE.,547b791c-f00d-4ffd-820c-b380b2c8ec41,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:45.840909,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24845,7 +24845,7 @@ class Problem { }",190.91330000082962,,Completions generated successfully. Ready for display in your IDE.,0c2c4a44-55d2-4a59-99b7-0dcd0f599f0e,1,deepseek-ai/deepseek-coder-1.3b-base," case 'e': case 'y': -",163,0.9843 +",163,0.9843 2025-09-22T13:58:46.030447,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -24895,7 +24895,7 @@ function prime_fib(n){ fibNum++; }",},189.73259999620495,,Completions generated successfully. Ready for display in your IDE.,f6412012-666f-4a01-8f7d-43034a8f1257,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(fibNum - 1); -",161,0.9583 +",161,0.9583 2025-09-22T13:58:46.249960,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -24919,7 +24919,7 @@ function vowels_count(s){ } ",219.4740000049933,,Completions generated successfully. Ready for display in your IDE.,3f4ce031-8ab9-4112-8642-5b2d7965cb84,1,deepseek-ai/deepseek-coder-1.3b-base," const vowels = ['a','e','i','o','u']; -",196,0.9542 +",196,0.9542 2025-09-22T13:58:46.320931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24966,7 +24966,7 @@ class Problem { return l_s; } }",70.652100002917,,Completions generated successfully. Ready for display in your IDE.,c5311bfe-c82c-4185-8635-72a5d99c5fb8,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,1.0 +",46,1.0 2025-09-22T13:58:46.448351,[1],"//Given an array arr of integers, find the minimum number of elements that // need to be changed to make the array palindromic. A palindromic array is an array that // is read the same backwards and forwards. In one change, you can change one element to any other element. @@ -24985,7 +24985,7 @@ function smallest_change(arr){ } return changes; }",126.79790000402136,,Completions generated successfully. Ready for display in your IDE.,57e5aa18-610f-4a9b-8cc4-118c4790e464,1,deepseek-ai/deepseek-coder-1.3b-base," let changes = 0; -",104,0.957 +",104,0.957 2025-09-22T13:58:46.539400,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -24996,7 +24996,7 @@ def all_prefixes(string: str) -> List[str]: prefixes = [] for i in range(len(string)): prefixes.append(string[:i+1])",,91.48909999930765,,Completions generated successfully. Ready for display in your IDE.,01dd7dec-015e-4191-a37b-0e94de67ac4e,1,deepseek-ai/deepseek-coder-1.3b-base," - return prefixes",69,0.9881 + return prefixes",69,0.9881 2025-09-22T13:58:46.695644,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -25016,7 +25016,7 @@ function sum_product(numbers){ ",155.33520000462886,,Completions generated successfully. Ready for display in your IDE.,8ce967bc-3fae-41f2-92c5-6a9e4a66e100,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... -",131,0.9213 +",131,0.9213 2025-09-22T13:58:46.783189,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -25045,7 +25045,7 @@ function is_simple_power(x, n){ } ",87.30240000295453,,Completions generated successfully. Ready for display in your IDE.,01fbbc23-64b3-4eea-8946-c44383a1abb0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:46.966158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25087,7 +25087,7 @@ class Problem { } }",183.19599999813363,,Completions generated successfully. Ready for display in your IDE.,dfb94214-842a-4f96-949d-58fe7ea51c94,1,deepseek-ai/deepseek-coder-1.3b-base," if (c2 < 0) break; -",152,1.0 +",152,1.0 2025-09-22T13:58:47.123173,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -25105,7 +25105,7 @@ class Problem { raise ValueError(""modp: invalid p"") if n < 0: raise ValueError(""modp: invalid n"")",,156.63019999919925,,Completions generated successfully. Ready for display in your IDE.,cf6d6abd-cba4-49b3-aab4-1e5a710e8dc9,1,deepseek-ai/deepseek-coder-1.3b-base," - return pow(2, n, p)",133,1.0 + return pow(2, n, p)",133,1.0 2025-09-22T13:58:47.211847,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -25135,7 +25135,7 @@ function sort_third(l){ return x; }",88.16859999933513,,Completions generated successfully. Ready for display in your IDE.,d766edf9-1957-4587-9fdf-06114ae40d23,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:47.310993,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25203,7 +25203,7 @@ function do_algebra(operator, operand){ } ",99.29729999566916,,Completions generated successfully. Ready for display in your IDE.,449e281e-4f6c-4905-a6aa-716223445732,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,0.9832 +",72,0.9832 2025-09-22T13:58:47.406691,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -25241,7 +25241,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",95.32689999468857,,Completions generated successfully. Ready for display in your IDE.,7a604917-dc49-4292-89bc-c692518b595a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:47.543618,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25283,7 +25283,7 @@ class Problem { return sb.toString(); } }",136.13059999624966,,Completions generated successfully. Ready for display in your IDE.,c8317808-0ad6-4c7a-9d92-31725a3e4d78,1,deepseek-ai/deepseek-coder-1.3b-base," boolean letterNotFound = true; -",111,1.0 +",111,1.0 2025-09-22T13:58:47.683649,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25361,7 +25361,7 @@ class Problem { } }",140.06169999629492,,Completions generated successfully. Ready for display in your IDE.,b3f39fac-4823-4bcd-9340-da7c877e428c,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""x""; -",113,1.0 +",113,1.0 2025-09-22T13:58:47.884165,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25405,7 +25405,7 @@ class Problem { } }",200.45180000306573,,Completions generated successfully. Ready for display in your IDE.,5c352d55-e64e-49cb-84f1-d2c33d0ab8e8,1,deepseek-ai/deepseek-coder-1.3b-base," public long getLastDigit(long n) { -",173,1.0 +",173,1.0 2025-09-22T13:58:48.577448,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25435,7 +25435,7 @@ class Problem { if (remainder == 0) { isPrime = false; - ",669,0.9489 + ",669,0.9489 2025-09-22T13:58:48.745736,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25456,7 +25456,7 @@ class Problem { } }",167.24989999784157,,Completions generated successfully. Ready for display in your IDE.,b20335b1-9bde-471e-ad7d-ebc319f54315,1,deepseek-ai/deepseek-coder-1.3b-base," .filter(l2::contains) -",142,0.9954 +",142,0.9954 2025-09-22T13:58:49.000333,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -25490,7 +25490,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",254.72780000563944,,Completions generated successfully. Ready for display in your IDE.,f332f814-aa34-4322-9110-d17d98ef26ca,1,deepseek-ai/deepseek-coder-1.3b-base," elif abs(numbers[i] - numbers[j]) == temp: -",228,1.0 +",228,1.0 2025-09-22T13:58:49.068020,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25557,7 +25557,7 @@ function do_algebra(operator, operand){ return answer; } ",67.29020000057062,,Completions generated successfully. Ready for display in your IDE.,497131bd-b032-42b6-b922-1918ffb27800,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T13:58:49.204153,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -25579,7 +25579,7 @@ function anti_shuffle(s){ result.push(temp.join("""")); });",},136.4735000024666,,Completions generated successfully. Ready for display in your IDE.,0dea1e00-396b-4c77-b4b6-40e6197e4a2b,1,deepseek-ai/deepseek-coder-1.3b-base," return result.join("" "") -",113,0.9713 +",113,0.9713 2025-09-22T13:58:49.291581,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -25611,7 +25611,7 @@ function correct_bracketing(brackets){ } }",86.87970000028145,,Completions generated successfully. Ready for display in your IDE.,ed37007f-a535-4343-99ce-c4afe0ce8cf6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:49.387849,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -25661,7 +25661,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",95.86829999898328,,Completions generated successfully. Ready for display in your IDE.,a984b1b5-21da-48dd-b700-03e091f30417,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:49.543966,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -25705,7 +25705,7 @@ def pluck(arr: List[int]) -> List[int]: ",155.9516000052099,,Completions generated successfully. Ready for display in your IDE.,904a82f9-8dc0-4d7b-b5e0-214d6de6e4b1,1,deepseek-ai/deepseek-coder-1.3b-base," return [] else: - ",130,1.0 + ",130,1.0 2025-09-22T13:58:49.663406,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -25734,7 +25734,7 @@ function split_words(txt){ } }",119.28250000346452,,Completions generated successfully. Ready for display in your IDE.,d54b8ef9-2fc1-472d-a738-7357a86de4da,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",91,1.0 +",91,1.0 2025-09-22T13:58:49.763920,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25776,7 +25776,7 @@ class Problem { }); return result;",},100.04000000481028,,Completions generated successfully. Ready for display in your IDE.,74ef9307-4a37-4ab7-b45b-4a1a3111939c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:50.509525,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25848,7 +25848,7 @@ function do_algebra(operator, operand){ operand.splice(count + 1, 1); operator.splice(count, 1); count = 0; - do_",685,1.0 + do_",685,1.0 2025-09-22T13:58:50.571680,[1],"//Checks if given string is a palindrome // >>> is_palindrome("""") // true @@ -25860,7 +25860,7 @@ function do_algebra(operator, operand){ // false function is_palindrome(text){ return text == text.split("""").reverse().join("""");",,62.08510000578826,,Completions generated successfully. Ready for display in your IDE.,50c32fb2-6aa4-4ae5-aa55-26042f10ec70,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:50.725742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25892,7 +25892,7 @@ class Problem { } }",153.75619999394985,,Completions generated successfully. Ready for display in your IDE.,9c9ac3b2-051d-4299-8d29-d9b12ebd5fbc,1,deepseek-ai/deepseek-coder-1.3b-base," while (first < last) { -",129,1.0 +",129,1.0 2025-09-22T13:58:51.420021,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25917,7 +25917,7 @@ class Problem { private static final Scanner scan = new Scanner(System.in); public static void main(String[] args){ int t = scan.nextInt(); - for(int i=0;ismallest && (isNaN(parseInt(lst[i]))|| parseInt(lst[i])>=0))&& !isNaN(parseInt(lst[i])) ){ -",587,0.9695 +",587,0.9695 2025-09-22T13:58:52.935805,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -26079,7 +26079,7 @@ function digits(n){ } return prod;"," ",77.08300000376767,,Completions generated successfully. Ready for display in your IDE.,a62e7a5d-fd27-49b8-8bf1-c0a45f59987f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:58:53.096759,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -26111,7 +26111,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",160.73840000171913,,Completions generated successfully. Ready for display in your IDE.,1cd738df-569a-4c3a-8965-3342cdb753e3,1,deepseek-ai/deepseek-coder-1.3b-base," most_count = letters[letter] -",135,1.0 +",135,1.0 2025-09-22T13:58:53.191337,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26153,7 +26153,7 @@ class Problem { } }",94.40680000261636,,Completions generated successfully. Ready for display in your IDE.,9c2e763b-fa1c-4dc8-8830-b6d78dc3e534,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:53.449353,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26185,7 +26185,7 @@ class Problem { } }",257.80179999856045,,Completions generated successfully. Ready for display in your IDE.,f181d517-09b6-45d2-99df-e19fa642a9db,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < numbers.size(); i++) { -",232,1.0 +",232,1.0 2025-09-22T13:58:53.543925,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -26224,7 +26224,7 @@ function find_max(words){ return max_unique_word; }",94.70140000485117,,Completions generated successfully. Ready for display in your IDE.,cba2d4ee-5733-48e9-827d-76f54f0f3ac4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:53.615931,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -26250,7 +26250,7 @@ function string_xor(a, b){ } result = arrayXOR.join(''); return result;",,71.54750000336207,,Completions generated successfully. Ready for display in your IDE.,8372c1f6-ff52-457f-9ffe-c4bd8d2a7d36,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:53.845299,[1],"from typing import List def mean_absolute_deviation(numbers: List[float]) -> float: @@ -26266,7 +26266,7 @@ def mean_absolute_deviation(numbers: List[float]) -> float: absolute_deviations = [abs(x - mean) for x in numbers]"," print(mean_absolute_deviation([1.0, 2.0, 3.0, 4.0]))",228.80840000289027,,Completions generated successfully. Ready for display in your IDE.,e725a90a-7557-4c5d-8d6e-b524fbdfcb10,1,deepseek-ai/deepseek-coder-1.3b-base," return sum(absolute_deviations) / len(numbers) -",206,0.9911 +",206,0.9911 2025-09-22T13:58:54.106200,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26306,7 +26306,7 @@ class Problem { return words.get(0); } }",260.62209999508923,,Completions generated successfully. Ready for display in your IDE.,bd61114e-db83-4360-853b-bed635922c30,1,deepseek-ai/deepseek-coder-1.3b-base," Map wordsToUniqueCharacters = new HashMap<>(); -",232,1.0 +",232,1.0 2025-09-22T13:58:54.170030,[1],"//Filter an input array of strings only for ones that start with a given prefix. // >>> filter_by_prefix([], ""a"") // [] @@ -26316,7 +26316,7 @@ function filter_by_prefix(strings, prefix){ return strings.filter( function(str){ return str.startsWith(prefix); });",,63.945500005502254,,Completions generated successfully. Ready for display in your IDE.,78d42d16-c6bb-45ed-8532-10c74bc828ce,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:54.276163,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26364,7 +26364,7 @@ class Problem { return new ArrayList(); }",},105.4803000006359,,Completions generated successfully. Ready for display in your IDE.,d0c301e0-c1d8-4b02-a07e-aa79d1c2360e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:58:54.334734,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -26391,7 +26391,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: min2 = lst[i] return min2 if min1 != min2 else None ",58.20970000058878,,Completions generated successfully. Ready for display in your IDE.,c2c3fde2-9930-427d-a7cc-9cd3d9a67025,1,deepseek-ai/deepseek-coder-1.3b-base," -",35,1.0 +",35,1.0 2025-09-22T13:58:54.464036,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -26427,7 +26427,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: ",129.96530000236817,,Completions generated successfully. Ready for display in your IDE.,fe8ed2fa-5f54-4824-9af7-68a1a0963d1f,1,deepseek-ai/deepseek-coder-1.3b-base," 9: 'ix', -",104,1.0 +",104,1.0 2025-09-22T13:58:54.651841,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -26454,7 +26454,7 @@ def find_max(words: List[str]) -> str: return max_word ",186.75810000422643,,Completions generated successfully. Ready for display in your IDE.,cfd869b3-e9d4-4c87-804e-a745cf0ff5fc,1,deepseek-ai/deepseek-coder-1.3b-base," unique_num = len(set(word)) -",163,1.0 +",163,1.0 2025-09-22T13:58:54.758505,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -26466,7 +26466,7 @@ def find_max(words: List[str]) -> str: return count ",106.97939999954542,,Completions generated successfully. Ready for display in your IDE.,5a8d10b6-3d8c-45b9-bc88-e243b61e286f,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",83,1.0 +",83,1.0 2025-09-22T13:58:54.907910,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26498,7 +26498,7 @@ class Problem { }"," } }",149.11730000312673,,Completions generated successfully. Ready for display in your IDE.,862c9fcb-c36e-40a1-ba8b-a1ae8a38470d,1,deepseek-ai/deepseek-coder-1.3b-base," return isPalindrome; -",125,1.0 +",125,1.0 2025-09-22T13:58:55.066717,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -26556,7 +26556,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.73580000334186,,Completions generated successfully. Ready for display in your IDE.,1aac02c5-cc91-4d32-a35d-3c381b2dc47c,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('C-') -",132,1.0 +",132,1.0 2025-09-22T13:58:55.124260,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26633,7 +26633,7 @@ class Problem { } return res; } -}",57.02439999731723,,Completions generated successfully. Ready for display in your IDE.,651d16af-90c5-485c-8626-adbd0db3be67,1,deepseek-ai/deepseek-coder-1.3b-base,,30,1.0 +}",57.02439999731723,,Completions generated successfully. Ready for display in your IDE.,651d16af-90c5-485c-8626-adbd0db3be67,1,deepseek-ai/deepseek-coder-1.3b-base,,30,1.0 2025-09-22T13:58:55.239434,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26664,7 +26664,7 @@ class Problem { } }",115.03480000101263,,Completions generated successfully. Ready for display in your IDE.,92c77b4a-6245-42fa-8ac4-9aab7f63b8b4,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",89,0.9286 +",89,0.9286 2025-09-22T13:58:55.541916,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -26681,7 +26681,7 @@ function how_many_times(string, substring){ }",302.6100999995833,,Completions generated successfully. Ready for display in your IDE.,223bd8a6-7110-4dbb-a691-bdc954f048dd,1,deepseek-ai/deepseek-coder-1.3b-base," i = string.indexOf(substring, i); i += substring.length; -",279,0.996 +",279,0.996 2025-09-22T13:58:56.234936,[1],"def sort_numbers(numbers: str) -> str: """""" Input is a space-delimited string of numberals from 'zero' to 'nine'. Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. @@ -26695,7 +26695,7 @@ function how_many_times(string, substring){ return res.strip() ",692.2431999992114,,Completions generated successfully. Ready for display in your IDE.,2c6643fe-6c14-4698-8468-cc3515ef59a2,1,deepseek-ai/deepseek-coder-1.3b-base," - mapper = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine':",668,0.9952 + mapper = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine':",668,0.9952 2025-09-22T13:58:56.348177,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26734,7 +26734,7 @@ class Problem { } }",112.69959999481216,,Completions generated successfully. Ready for display in your IDE.,dd442640-964b-4e30-b4b5-30588a5e3325,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",89,1.0 +",89,1.0 2025-09-22T13:58:56.813267,[1],"//Return a greatest common divisor of two integers a and b // >>> greatest_common_divisor(3, 5) // 1 @@ -26752,7 +26752,7 @@ function greatest_common_divisor(a, b){ for (i = 1; i <= min/2; ++i ){ if ((a%i==0) && (b%i==0)) return i } -",443,0.9674 +",443,0.9674 2025-09-22T13:58:56.900833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26782,7 +26782,7 @@ class Problem { } }",87.90840000438038,,Completions generated successfully. Ready for display in your IDE.,c862ace2-31bf-41c9-97d9-a8e96885c15b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:57.126112,[1],"//Return a string containing space-delimited numbers starting from 0 upto n inclusive. // >>> string_sequence(0) // ""0"" @@ -26795,7 +26795,7 @@ function string_sequence(n){ } ",225.03190000134055,,Completions generated successfully. Ready for display in your IDE.,f21c9b96-eae9-4a33-9d50-a8412629bc2b,1,deepseek-ai/deepseek-coder-1.3b-base," for (let i = 0; i <= n ; i++ ){ -",203,0.9619 +",203,0.9619 2025-09-22T13:58:57.396817,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -26810,7 +26810,7 @@ function string_sequence(n){ if n < 10: return n if n % 2 == 1 else 0 d, m = divmod(n, 10)",,269.9012000011862,,Completions generated successfully. Ready for display in your IDE.,92876fe1-6e5c-4d19-8318-ba6014d8f3ef,1,deepseek-ai/deepseek-coder-1.3b-base," - return (m if m % 2 == 1 else 0) + digits(d)",247,0.9861 + return (m if m % 2 == 1 else 0) + digits(d)",247,0.9861 2025-09-22T13:58:57.546098,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26850,7 +26850,7 @@ class Problem { } }",149.54649999708636,,Completions generated successfully. Ready for display in your IDE.,5a89816e-0317-4beb-a634-17ff74a38d67,1,deepseek-ai/deepseek-coder-1.3b-base," arr.add(2l); -",126,0.9899 +",126,0.9899 2025-09-22T13:58:57.710100,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -26922,7 +26922,7 @@ function minPath(grid, k){ return path; }",163.79919999599224,,Completions generated successfully. Ready for display in your IDE.,d16b75bc-b4c2-436a-bd5b-a341bbad6626,1,deepseek-ai/deepseek-coder-1.3b-base," if(y > 0){ -",135,1.0 +",135,1.0 2025-09-22T13:58:57.826312,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -26948,7 +26948,7 @@ function fibfib(n){ ",115.57000000175321,,Completions generated successfully. Ready for display in your IDE.,12e6d977-3f61-4336-a7f8-69a444d0d07e,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",93,1.0 +",93,1.0 2025-09-22T13:58:57.914824,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26976,7 +26976,7 @@ class Problem { } }",88.58049999980722,,Completions generated successfully. Ready for display in your IDE.,f4af1f8e-b625-4844-8c9c-b854b10bfe55,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:58.040365,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27014,7 +27014,7 @@ class Problem { } }",125.88580000010552,,Completions generated successfully. Ready for display in your IDE.,53da4a36-a15e-4268-a6f9-3c9d90958aea,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",97,1.0 +",97,1.0 2025-09-22T13:58:58.166717,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -27033,7 +27033,7 @@ function sum_product(numbers){ } ",125.1923000017996,,Completions generated successfully. Ready for display in your IDE.,259cea27-9e79-454f-82dd-677292ff59cb,1,deepseek-ai/deepseek-coder-1.3b-base," let product = 1; -",104,1.0 +",104,1.0 2025-09-22T13:58:58.283548,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27061,7 +27061,7 @@ class Problem { } }",116.73180000070715,,Completions generated successfully. Ready for display in your IDE.,339cbd78-665b-4ba4-b92e-bb6812844238,1,deepseek-ai/deepseek-coder-1.3b-base," boredoms++; -",93,1.0 +",93,1.0 2025-09-22T13:58:58.369163,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -27078,7 +27078,7 @@ function largest_prime_factor(n){ return result; }",85.46989999740617,,Completions generated successfully. Ready for display in your IDE.,378a731f-9937-45e1-b6a8-6341333fb709,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:58.448202,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -27107,7 +27107,7 @@ function get_row(lst, x){ return coordinates.sort(function(a, b){ return a[0] - b[0] || b[1] - a[1]; });",,79.17239999369485,,Completions generated successfully. Ready for display in your IDE.,c843f4d8-7a49-4cf7-b140-fe064897051a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",55,1.0 +}",55,1.0 2025-09-22T13:58:58.567373,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27169,7 +27169,7 @@ class Problem { } }",118.32039999717381,,Completions generated successfully. Ready for display in your IDE.,c3d42b62-59ea-4e3d-9dff-aee82f30c952,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:58:58.695978,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27212,7 +27212,7 @@ class Problem { }"," } }",128.81689999630908,,Completions generated successfully. Ready for display in your IDE.,7d3d4afd-00ad-49cd-bd10-01015b5c8107,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",98,1.0 +",98,1.0 2025-09-22T13:58:58.918908,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27290,7 +27290,7 @@ class Problem { } }",222.77349999785656,,Completions generated successfully. Ready for display in your IDE.,2931f089-4935-4b22-9018-ec12616281db,1,deepseek-ai/deepseek-coder-1.3b-base," if (number < 1 || number > 3999) -",195,0.9922 +",195,0.9922 2025-09-22T13:58:58.983101,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -27323,7 +27323,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.37789999815868,,Completions generated successfully. Ready for display in your IDE.,007e375a-480f-4a1d-ab06-842a778acc5d,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:58:59.100843,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -27391,7 +27391,7 @@ function do_algebra(operator, operand){ } ",117.53260000114096,,Completions generated successfully. Ready for display in your IDE.,2674aca8-e546-4595-95ec-ff9ac76b5939,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",83,1.0 +",83,1.0 2025-09-22T13:58:59.220187,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -27414,7 +27414,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",119.26559999847086,,Completions generated successfully. Ready for display in your IDE.,dff9adea-e0b8-4261-a08f-7d413895df11,1,deepseek-ai/deepseek-coder-1.3b-base," even_count += 1 -",98,1.0 +",98,1.0 2025-09-22T13:58:59.461771,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27441,7 +27441,7 @@ class Problem { } }",241.68140000256244,,Completions generated successfully. Ready for display in your IDE.,e0af103f-f5b0-4ab5-9167-7e7deba93951,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(n - 1) + fib(n - 2); -",217,1.0 +",217,1.0 2025-09-22T13:58:59.560162,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27490,7 +27490,7 @@ class Problem { } }",97.70220000064,,Completions generated successfully. Ready for display in your IDE.,1b60bd84-ec2d-43c2-9bbf-95b68dd552fd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:59:00.088064,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -27530,7 +27530,7 @@ function pluck(arr){ result = [[arr[i], i]] } else { (index === -1 && result[0][1] > i)? null : result.push([arr[i], i]); -",503,0.9315 +",503,0.9315 2025-09-22T13:59:00.306699,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27563,7 +27563,7 @@ class Problem { } }",218.17490000103135,,Completions generated successfully. Ready for display in your IDE.,96335bac-351e-4be9-99d8-950b1b2232c2,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(arr); int first = 0; -",193,0.9555 +",193,0.9555 2025-09-22T13:59:00.408656,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -27604,7 +27604,7 @@ def move_one_ball(arr: List[int]) -> bool: elif arr[i-1] < arr[i]:"," return True ",101.32290000183275,,Completions generated successfully. Ready for display in your IDE.,882a207c-5e73-44b8-9477-953def72e951,1,deepseek-ai/deepseek-coder-1.3b-base," continue -",69,0.9848 +",69,0.9848 2025-09-22T13:59:00.540516,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -27632,7 +27632,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i return (largest_negative, smallest_positive) ",131.66640000417829,,Completions generated successfully. Ready for display in your IDE.,f57d4881-2169-43aa-ac0f-ea648ad4572a,1,deepseek-ai/deepseek-coder-1.3b-base," for num in lst: -",109,1.0 +",109,1.0 2025-09-22T13:59:00.652850,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27668,7 +27668,7 @@ class Problem { }"," } }",112.50300000392599,,Completions generated successfully. Ready for display in your IDE.,b22ef6e8-ee05-4f14-b08f-248ad3eda1ab,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",88,1.0 +",88,1.0 2025-09-22T13:59:00.767427,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27700,7 +27700,7 @@ class Problem { }"," } }",113.88770000485238,,Completions generated successfully. Ready for display in your IDE.,2c535526-8b7a-4a44-a29b-b01dfe2495b3,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",88,1.0 +",88,1.0 2025-09-22T13:59:00.971455,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27762,7 +27762,7 @@ class Problem { } }",203.65209999727085,,Completions generated successfully. Ready for display in your IDE.,9ef29577-9508-43ff-aa3c-501b89ed9205,1,deepseek-ai/deepseek-coder-1.3b-base," double number = Math.sqrt(item); -",174,0.9942 +",174,0.9942 2025-09-22T13:59:01.059572,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27805,7 +27805,7 @@ class Problem { } }",88.33040000172332,,Completions generated successfully. Ready for display in your IDE.,da4365a2-e8dd-4535-9d46-9016be029e33,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:59:01.230412,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -27838,7 +27838,7 @@ class Problem { ",170.87800000444986,,Completions generated successfully. Ready for display in your IDE.,00c6af75-b0f7-41b9-9fe4-82955c8419d4,1,deepseek-ai/deepseek-coder-1.3b-base," while (i < n - 2): -",146,0.9912 +",146,0.9912 2025-09-22T13:59:01.386085,[1],"//You are given two intervals, // where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). // The given intervals are closed which means that the interval (start, end) @@ -27868,7 +27868,7 @@ function intersection(interval1, interval2){ } ",154.6546000026865,,Completions generated successfully. Ready for display in your IDE.,20ce2dda-3bc3-4388-bfa3-14bc0adb51fc,1,deepseek-ai/deepseek-coder-1.3b-base," if ((end < start)) { -",129,0.957 +",129,0.957 2025-09-22T13:59:01.479483,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -27904,7 +27904,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",93.28249999816762,,Completions generated successfully. Ready for display in your IDE.,b84dc680-f411-4f91-9e8e-9a42b63302e8,1,deepseek-ai/deepseek-coder-1.3b-base," break -",67,1.0 +",67,1.0 2025-09-22T13:59:01.655528,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -27938,7 +27938,7 @@ function count_nums(arr){ return count; }",176.23530000128085,,Completions generated successfully. Ready for display in your IDE.,910142fc-aaf5-4dee-a7b4-b48cb9f691ea,1,deepseek-ai/deepseek-coder-1.3b-base," sum += Number(str[j]); -",153,1.0 +",153,1.0 2025-09-22T13:59:01.856883,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -27964,7 +27964,7 @@ function correct_bracketing(brackets){ } ",201.03759999619797,,Completions generated successfully. Ready for display in your IDE.,20e1c5f2-7ee7-4de9-950d-47ad09aba242,1,deepseek-ai/deepseek-coder-1.3b-base," if (brackets[i] === ""<"") { -",176,1.0 +",176,1.0 2025-09-22T13:59:02.055699,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -27986,7 +27986,7 @@ function correct_bracketing(brackets){ return n * special_factorial(n-1) * factorial(n-1) ",198.5930999944685,,Completions generated successfully. Ready for display in your IDE.,4c830b36-8b9c-48a6-8a52-b90ac41728b6,1,deepseek-ai/deepseek-coder-1.3b-base," - return n * special_factorial(n-1)",175,0.9536 + return n * special_factorial(n-1)",175,0.9536 2025-09-22T13:59:02.201147,[1],"//Given a non-empty array of integers, return the sum of all of the odd elements that are in even positions. // Examples // >>> solution([5, 8, 7, 1]) @@ -28005,7 +28005,7 @@ function solution(lst){ return result; }",145.04569999553496,,Completions generated successfully. Ready for display in your IDE.,18865d8f-d96e-4ac8-a215-710a00771151,1,deepseek-ai/deepseek-coder-1.3b-base," result += lst[i]; -",123,1.0 +",123,1.0 2025-09-22T13:59:02.316818,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -28082,7 +28082,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",115.74829999881331,,Completions generated successfully. Ready for display in your IDE.,766694b0-c85c-480c-9533-7c92958c4383,1,deepseek-ai/deepseek-coder-1.3b-base," return a -",88,1.0 +",88,1.0 2025-09-22T13:59:02.416344,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28126,7 +28126,7 @@ class Problem { } }",98.7950000053388,,Completions generated successfully. Ready for display in your IDE.,f4538971-db2c-4334-bd29-c97356c876be,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:02.570023,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -28181,7 +28181,7 @@ function by_length(arr){ return arr3; }",153.73460000409978,,Completions generated successfully. Ready for display in your IDE.,e6b3e961-82b1-4e6b-93bf-35605d7847ce,1,deepseek-ai/deepseek-coder-1.3b-base," else if(num == 3){ -",130,1.0 +",130,1.0 2025-09-22T13:59:02.748166,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -28200,7 +28200,7 @@ function by_length(arr){ return 1 if n == 0 else 2**(n % (p-1)) % p ",178.22610000439454,,Completions generated successfully. Ready for display in your IDE.,e25299df-ae29-4cd8-92db-c1b07ef659eb,1,deepseek-ai/deepseek-coder-1.3b-base," raise ValueError(""modp: invalid p"") -",156,1.0 +",156,1.0 2025-09-22T13:59:02.858114,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -28218,7 +28218,7 @@ function by_length(arr){ ",109.40480000135722,,Completions generated successfully. Ready for display in your IDE.,13b6a1a9-9676-4014-87e0-5c26e9708efe,1,deepseek-ai/deepseek-coder-1.3b-base," factor = 2 -",84,1.0 +",84,1.0 2025-09-22T13:59:03.025297,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -28244,7 +28244,7 @@ def smallest_change(arr: List[int]) -> int: return count ",166.6038999974262,,Completions generated successfully. Ready for display in your IDE.,88ac55b1-a85a-481e-bdaa-dcc0e008918e,1,deepseek-ai/deepseek-coder-1.3b-base," end = len(arr) - 1 -",142,1.0 +",142,1.0 2025-09-22T13:59:03.413260,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28306,7 +28306,7 @@ class Problem { } }",387.78959999763174,,Completions generated successfully. Ready for display in your IDE.,b8f3fb6d-bfde-4ec6-a8aa-b3eb1e079420,1,deepseek-ai/deepseek-coder-1.3b-base," while ((int)(Math.pow(10,(double)counter+1)) <= Math.sqrt(number)) { -",341,0.9408 +",341,0.9408 2025-09-22T13:59:03.643938,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -28334,7 +28334,7 @@ function strange_sort_list(lst){ }",230.28469999553636,,Completions generated successfully. Ready for display in your IDE.,a356197b-2d41-42fc-b67a-6f899f68173e,1,deepseek-ai/deepseek-coder-1.3b-base," //Sort list using Math.min() & Math.max(). -",206,0.9076 +",206,0.9076 2025-09-22T13:59:03.721931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28375,7 +28375,7 @@ class Problem { return result; } }",78.41850000113482,,Completions generated successfully. Ready for display in your IDE.,3bae6d64-87f9-47e4-9282-902ecae54106,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,1.0 + ",48,1.0 2025-09-22T13:59:04.162768,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -28425,7 +28425,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",440.2556999993976,,Completions generated successfully. Ready for display in your IDE.,db23f701-0c9b-4cfb-8bb8-b729a7ad9f7d,1,deepseek-ai/deepseek-coder-1.3b-base," if(isPrime(fib(fibNum)) == true && !Number.isInteger(Math.sqrt(fib(fibNum))) ) { -",416,0.9565 +",416,0.9565 2025-09-22T13:59:04.249043,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -28455,7 +28455,7 @@ function parse_nested_parens(paren_string){ ",86.17880000383593,,Completions generated successfully. Ready for display in your IDE.,1a6ca84d-35c0-4f62-889f-520352308ef6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:04.345481,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28494,7 +28494,7 @@ class Problem { } }",95.88459999940824,,Completions generated successfully. Ready for display in your IDE.,c64cf5a1-3545-4b08-a622-5e4fb166f11d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:59:04.688119,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -28521,7 +28521,7 @@ def smallest_change(arr: List[int]) -> int: ",342.58630000113044,,Completions generated successfully. Ready for display in your IDE.,781d8ae9-e4a6-4bec-b329-5cf47377a733,1,deepseek-ai/deepseek-coder-1.3b-base," if arr[start] > arr[end]: count += (arr[start] - arr[end]) + 1 -",319,0.9755 +",319,0.9755 2025-09-22T13:59:04.906737,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -28549,7 +28549,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i return (largest_negative, smallest_positive) ",218.24480000213953,,Completions generated successfully. Ready for display in your IDE.,3be8a53e-b2c3-4820-9b5e-e760bf68aa02,1,deepseek-ai/deepseek-coder-1.3b-base," if smallest_positive is None or num < smallest_positive: -",193,1.0 +",193,1.0 2025-09-22T13:59:05.044029,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -28578,7 +28578,7 @@ function f(n){ return result; }",136.9838000027812,,Completions generated successfully. Ready for display in your IDE.,574d9b19-d8a1-48bd-8d34-ca3bcb9f86c0,1,deepseek-ai/deepseek-coder-1.3b-base," var factorial = 1; -",113,1.0 +",113,1.0 2025-09-22T13:59:05.739459,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -28613,7 +28613,7 @@ function closest_integer(value){ const absValue = Math.abs(value); console.log('Abs Value', absValue); if(!isNaN(absValue)){ -",669,0.8937 +",669,0.8937 2025-09-22T13:59:05.792875,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -28637,7 +28637,7 @@ def sort_array(array: List[int]) -> List[int]: """""" if array: return sorted(array, reverse=True) if (array[0] + array[-1]) % 2 == 0 else sorted(array)",,53.631299997505266,,Completions generated successfully. Ready for display in your IDE.,fbcc3c37-8704-489f-a4c1-15e10e5ef017,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,0.9246 +",30,0.9246 2025-09-22T13:59:05.865545,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -28651,7 +28651,7 @@ function filter_by_substring(strings, substring){ } } return filtered;",,72.29659999575233,,Completions generated successfully. Ready for display in your IDE.,efcfd336-d83a-4642-bb18-13b0d6a5669f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:59:05.960126,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28689,7 +28689,7 @@ class Problem { } return result;",},93.91770000365796,,Completions generated successfully. Ready for display in your IDE.,481ae84f-264f-4592-b7ad-696fec27d03d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:06.046145,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28713,7 +28713,7 @@ class Problem { } }",86.20690000680042,,Completions generated successfully. Ready for display in your IDE.,8ceeb8be-958d-4eb3-8bf3-228172d1c6f8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:06.268946,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -28740,7 +28740,7 @@ function strange_sort_list(lst){ return output; }",222.25299999990966,,Completions generated successfully. Ready for display in your IDE.,2579581b-e9e4-4f9e-8e74-48de4e3c438d,1,deepseek-ai/deepseek-coder-1.3b-base," output.push(Math.max.apply(null, tmp)); -",198,1.0 +",198,1.0 2025-09-22T13:59:06.562305,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28783,7 +28783,7 @@ class Problem { }",293.2291000033729,,Completions generated successfully. Ready for display in your IDE.,69668df5-c80f-4ac1-ac84-9fa0710b346c,1,deepseek-ai/deepseek-coder-1.3b-base," else { return Integer.compare(s1.length(), s2.length()); } -",266,0.9551 +",266,0.9551 2025-09-22T13:59:06.730195,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -28838,7 +28838,7 @@ function by_length(arr){ return arr3; }",167.68379999848548,,Completions generated successfully. Ready for display in your IDE.,d404ca41-9c60-401e-8733-075976f1df25,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Six""); -",141,1.0 +",141,1.0 2025-09-22T13:59:06.920986,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28878,7 +28878,7 @@ class Problem { return array; } }",190.5996000059531,,Completions generated successfully. Ready for display in your IDE.,4c684b50-27f7-49d1-adec-f1febd0c88e1,1,deepseek-ai/deepseek-coder-1.3b-base," if(array == null || array.isEmpty()) { -",165,1.0 +",165,1.0 2025-09-22T13:59:07.061609,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28956,7 +28956,7 @@ class Problem { } }",140.19220000045607,,Completions generated successfully. Ready for display in your IDE.,23f1a188-4d24-41d4-a7ed-6eda6d17eea5,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""xc""; -",113,1.0 +",113,1.0 2025-09-22T13:59:07.170411,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -28985,7 +28985,7 @@ function is_simple_power(x, n){ } ",108.81479999807198,,Completions generated successfully. Ready for display in your IDE.,c2bfef67-9e1d-42b4-b0d9-61e40738cec3,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",84,1.0 +",84,1.0 2025-09-22T13:59:07.348220,[1],"//Given a non-empty array of integers arr and an integer k, return // the sum of the elements with at most two digits from the first k elements of arr. // Example: @@ -29005,7 +29005,7 @@ function add_elements(arr, k){ break; } } -",155,0.9946 +",155,0.9946 2025-09-22T13:59:07.499496,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -29035,7 +29035,7 @@ function words_in_sentence(sentence){ }).join(' '); }",150.75390000129119,,Completions generated successfully. Ready for display in your IDE.,cb32a51f-2643-4cc2-bdcc-8d8a3ca6eea0,1,deepseek-ai/deepseek-coder-1.3b-base," let isPrime = true; -",125,1.0 +",125,1.0 2025-09-22T13:59:07.658436,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -29093,7 +29093,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.1504999960307,,Completions generated successfully. Ready for display in your IDE.,fd487a76-1484-47fa-9381-2fcb0900a3a9,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('A-') -",132,1.0 +",132,1.0 2025-09-22T13:59:07.808380,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29125,7 +29125,7 @@ class Problem { } }",149.69589999964228,,Completions generated successfully. Ready for display in your IDE.,b4c9b42d-2a3b-4a47-8d84-cec1f006694d,1,deepseek-ai/deepseek-coder-1.3b-base," current_depth -= 1; -",126,1.0 +",126,1.0 2025-09-22T13:59:08.353551,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -29148,7 +29148,7 @@ function circular_shift(x, shift){ //if shift is greater than length of input, reverse it str = str.substring(len - shift); res = str + str.substring(0, len - shift); -",505,0.8737 +",505,0.8737 2025-09-22T13:59:08.643933,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -29168,7 +29168,7 @@ def sort_even(l: List[int]) -> List[int]: return l",290.4731000016909,,Completions generated successfully. Ready for display in your IDE.,e36bd28d-4c40-4bed-ac55-47ee7b68b249,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): if not (i % 2): - ",267,0.9983 + ",267,0.9983 2025-09-22T13:59:08.777317,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -29195,7 +29195,7 @@ def find_max(words: List[str]) -> str: return max_word ",133.04649999918183,,Completions generated successfully. Ready for display in your IDE.,4106d775-5cc1-406d-b915-e18ddf7f9807,1,deepseek-ai/deepseek-coder-1.3b-base," for word in words: -",105,1.0 +",105,1.0 2025-09-22T13:59:08.863598,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -29216,7 +29216,7 @@ function specialFilter(nums){ } ",86.2894999954733,,Completions generated successfully. Ready for display in your IDE.,825085a5-1583-4fd9-b62a-b2d09fbd46b5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,0.9242 +",62,0.9242 2025-09-22T13:59:09.127156,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -29244,7 +29244,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",262.91240000136895,,Completions generated successfully. Ready for display in your IDE.,af4f7015-0f2b-4d89-8d7b-1588bed48f61,1,deepseek-ai/deepseek-coder-1.3b-base," if n_uppercase - n_lowercase > best_strength: -",235,1.0 +",235,1.0 2025-09-22T13:59:09.266621,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29294,7 +29294,7 @@ class Problem { } }",139.5436000020709,,Completions generated successfully. Ready for display in your IDE.,dce0fc7b-3084-4655-b7dc-6ccd033475df,1,deepseek-ai/deepseek-coder-1.3b-base," switch (c) { -",111,1.0 +",111,1.0 2025-09-22T13:59:09.537488,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -29318,7 +29318,7 @@ D.1 Recursion Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",270.51139999821316,,Completions generated successfully. Ready for display in your IDE.,aa2bf01a-3561-4f02-8909-c4ec11458547,1,deepseek-ai/deepseek-coder-1.3b-base," vowel_set = {'a', 'e', 'i', 'o', 'u'} -",246,0.9734 +",246,0.9734 2025-09-22T13:59:09.593180,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -29343,7 +29343,7 @@ function separate_paren_groups(paren_string){ return groups; } ",56.05549999745563,,Completions generated successfully. Ready for display in your IDE.,5ac318aa-cb1b-468e-9b49-8cb695289ca5,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:09.749284,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -29366,7 +29366,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: return result ",156.00850000191713,,Completions generated successfully. Ready for display in your IDE.,be15f421-5ab8-4411-9d20-cb530a0c11fd,1,deepseek-ai/deepseek-coder-1.3b-base," for paren in parens: -",132,1.0 +",132,1.0 2025-09-22T13:59:09.830206,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -29437,7 +29437,7 @@ function minPath(grid, k){ } return path; }",80.51709999563172,,Completions generated successfully. Ready for display in your IDE.,34a1eafb-e5c8-4870-995a-f27ce7bb5155,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:59:09.928518,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -29468,7 +29468,7 @@ function same_chars(s0, s1){ ",97.51069999765605,,Completions generated successfully. Ready for display in your IDE.,1e003a7b-64bf-4570-8d6d-d3fee9d0e962,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",73,1.0 +",73,1.0 2025-09-22T13:59:10.627404,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -29514,7 +29514,7 @@ function compare_one(a, b){ if ((isNaN(parseFloat(a)) || !isFinite(a))) { console.log('Not an integer'); - }else if((isNaN(parseFloat(b)) || !isFinite(b)))",674,0.9511 + }else if((isNaN(parseFloat(b)) || !isFinite(b)))",674,0.9511 2025-09-22T13:59:10.744871,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29542,7 +29542,7 @@ class Problem { } }",117.42789999698289,,Completions generated successfully. Ready for display in your IDE.,1e378503-10d1-4e29-93f5-6e95168490b1,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",93,1.0 +",93,1.0 2025-09-22T13:59:10.886233,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -29569,7 +29569,7 @@ class Problem { return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",141.21639999939362,,Completions generated successfully. Ready for display in your IDE.,6c1fea92-9d80-449d-a0e5-5b546bd56c10,1,deepseek-ai/deepseek-coder-1.3b-base," if ch.isalpha(): -",114,0.9775 +",114,0.9775 2025-09-22T13:59:10.978464,[1],"from typing import List def specialFilter(nums: List[int]) -> int: @@ -29590,7 +29590,7 @@ def specialFilter(nums: List[int]) -> int: rev = int(str(num)[::-1]) if num % 2 != 0 and rev % 2 != 0: count += 1",,92.041000003519,,Completions generated successfully. Ready for display in your IDE.,9bea6dc5-f7a5-4aa6-ba06-f9f21bf61f0d,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",62,0.9596 + return count",62,0.9596 2025-09-22T13:59:11.063929,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -29616,7 +29616,7 @@ function correct_bracketing(brackets){ } ",84.55939999839757,,Completions generated successfully. Ready for display in your IDE.,e24043dd-653a-4027-abd9-ec047b3f9f22,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:11.150077,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29637,7 +29637,7 @@ class Problem { } return result.trim();",},86.30280000215862,,Completions generated successfully. Ready for display in your IDE.,d7476345-5d50-405c-b8e6-dd11fd874c7a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:11.330466,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29666,7 +29666,7 @@ class Problem { return primes.get(primes.size() - 1); } }",180.30719999660505,,Completions generated successfully. Ready for display in your IDE.,882baf26-2f5f-48e8-a762-f91b206b4049,1,deepseek-ai/deepseek-coder-1.3b-base," List primes = new ArrayList<>(); -",156,1.0 +",156,1.0 2025-09-22T13:59:11.402163,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -29697,7 +29697,7 @@ function skjkasdkd(lst){ sum += Number(digits[i]); } return sum;",,71.78340000245953,,Completions generated successfully. Ready for display in your IDE.,7af4e63c-6afe-4f8d-97bf-36abfeab24e1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:59:11.444581,[1],"//Return median of elements in the array l. // >>> median([3, 1, 2, 4, 5]) // 3 @@ -29711,7 +29711,7 @@ function median(l){ return sorted[middle]; } } -",41.493499993521255,,Completions generated successfully. Ready for display in your IDE.,75d53283-9ae8-48b2-88bf-db9252ee1b3a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5919 +",41.493499993521255,,Completions generated successfully. Ready for display in your IDE.,75d53283-9ae8-48b2-88bf-db9252ee1b3a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5919 2025-09-22T13:59:11.726684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29742,7 +29742,7 @@ class Problem { }",282.19639999588253,,Completions generated successfully. Ready for display in your IDE.,9e996d84-a979-4755-9bb9-47f09b82ca7e,1,deepseek-ai/deepseek-coder-1.3b-base," char c = s.charAt(j); if (c == '7') { -",259,1.0 +",259,1.0 2025-09-22T13:59:11.862054,[1],"from typing import List def get_positive(l: List[int]) -> List[int]: @@ -29757,7 +29757,7 @@ def get_positive(l: List[int]) -> List[int]: if i > 0:"," return p ",134.9019999979646,,Completions generated successfully. Ready for display in your IDE.,3f96db7a-b4e2-4bfc-951b-fd4e32aae8ed,1,deepseek-ai/deepseek-coder-1.3b-base," p.append(i) -",113,1.0 +",113,1.0 2025-09-22T13:59:11.918123,[1],"def flip_case(string: str) -> str: """""" For a given string, flip lowercase characters to uppercase and uppercase to lowercase. >>> flip_case('Hello') @@ -29770,7 +29770,7 @@ def get_positive(l: List[int]) -> List[int]: new_string += character.upper() return new_string -",55.99709999660263,,Completions generated successfully. Ready for display in your IDE.,54bdd415-e946-4fae-bfeb-a063d590c74a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +",55.99709999660263,,Completions generated successfully. Ready for display in your IDE.,54bdd415-e946-4fae-bfeb-a063d590c74a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:59:12.051432,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29808,7 +29808,7 @@ class Problem { } }",133.30409999616677,,Completions generated successfully. Ready for display in your IDE.,649609a8-ccb1-4233-9bb9-78587c129ca7,1,deepseek-ai/deepseek-coder-1.3b-base," int min = 0; -",109,1.0 +",109,1.0 2025-09-22T13:59:12.142133,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29841,7 +29841,7 @@ class Problem { } }",89.93499999633059,,Completions generated successfully. Ready for display in your IDE.,9123dc2f-b803-494f-82d6-f76302ace5a1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:12.226953,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -29861,7 +29861,7 @@ function unique_digits(x){ ",85.10259999457048,,Completions generated successfully. Ready for display in your IDE.,0f6ce1cd-4d29-4d63-94ca-cafb9cb244df,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:12.383786,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -29916,7 +29916,7 @@ function by_length(arr){ } }",},157.01419999822974,,Completions generated successfully. Ready for display in your IDE.,9519f486-6df7-43b7-979e-dfafd9cf97dd,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(arr3); -",130,1.0 +",130,1.0 2025-09-22T13:59:12.491327,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29946,7 +29946,7 @@ class Problem { }"," } }",106.57940000237431,,Completions generated successfully. Ready for display in your IDE.,c9a1ac0e-e834-4a17-a96f-cf0ec3033701,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",82,1.0 +",82,1.0 2025-09-22T13:59:12.669431,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29994,7 +29994,7 @@ class Problem { } }",178.57749999529915,,Completions generated successfully. Ready for display in your IDE.,338b769f-ea3b-44e0-811f-50de4e8dc937,1,deepseek-ai/deepseek-coder-1.3b-base," indexMax = indexPlanet1; -",149,1.0 +",149,1.0 2025-09-22T13:59:12.730823,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30022,7 +30022,7 @@ class Problem { if (n == 2) return 2;"," return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4); } }",60.838900004455354,,Completions generated successfully. Ready for display in your IDE.,8971f7b0-680e-47d2-ae51-99427e0a2be9,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T13:59:12.992532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30055,7 +30055,7 @@ class Problem { }",261.18549999955576,,Completions generated successfully. Ready for display in your IDE.,78794d56-97d8-4cbb-9204-4cab5b584886,1,deepseek-ai/deepseek-coder-1.3b-base," while ((left < right) && (isPalindrome == true)) { -",238,0.9772 +",238,0.9772 2025-09-22T13:59:13.055959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30075,7 +30075,7 @@ class Problem { } return 1; }",,63.48189999698661,,Completions generated successfully. Ready for display in your IDE.,70ad8bf8-57f4-439f-9ef5-13aac85a87b6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:13.329689,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30128,7 +30128,7 @@ class Problem { } }",273.6475999990944,,Completions generated successfully. Ready for display in your IDE.,c93c9780-ff80-46f5-b1fd-380ead1e5048,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < arr.size(); ++i) { -",247,1.0 +",247,1.0 2025-09-22T13:59:13.539080,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30154,7 +30154,7 @@ class Problem { } }",208.9759000009508,,Completions generated successfully. Ready for display in your IDE.,9d7b9783-3fb0-48bc-97de-b91913493081,1,deepseek-ai/deepseek-coder-1.3b-base," long y = Math.abs(b % 10); -",186,0.9578 +",186,0.9578 2025-09-22T13:59:13.704532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30194,7 +30194,7 @@ class Problem { } }",165.5077999967034,,Completions generated successfully. Ready for display in your IDE.,63d91502-218a-4050-a279-240cedb91851,1,deepseek-ai/deepseek-coder-1.3b-base," if (compareResult == 0) { -",140,1.0 +",140,1.0 2025-09-22T13:59:13.808004,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30233,7 +30233,7 @@ class Problem { } }",103.22879999876022,,Completions generated successfully. Ready for display in your IDE.,e75bb758-47a8-4d0b-97b4-c68af9d7007d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T13:59:13.901236,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -30259,7 +30259,7 @@ function string_xor(a, b){ return result; }",93.33769999648212,,Completions generated successfully. Ready for display in your IDE.,6ba7bfd8-eb02-447b-8354-5a933109694f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:13.956451,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -30281,7 +30281,7 @@ function string_xor(a, b){ """"""","Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. -""""""",54.90109999664128,,Completions generated successfully. Ready for display in your IDE.,33c94c3d-32c0-4068-865b-79f92c6cfc52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +""""""",54.90109999664128,,Completions generated successfully. Ready for display in your IDE.,33c94c3d-32c0-4068-865b-79f92c6cfc52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:14.077587,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30343,7 +30343,7 @@ class Problem { } }",120.36800000350922,,Completions generated successfully. Ready for display in your IDE.,ecda7b8d-16bb-43df-9c23-4abfa2920414,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:59:14.151835,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -30376,7 +30376,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",74.04449999739882,,Completions generated successfully. Ready for display in your IDE.,83eb826d-a6cf-4f6b-ac6d-17ab38db9807,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,0.9689 + ",48,0.9689 2025-09-22T13:59:14.332642,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30401,7 +30401,7 @@ class Problem { } }",181.07400000008056,,Completions generated successfully. Ready for display in your IDE.,fa5251b1-c384-4331-9877-c7d9fdd8cfd6,1,deepseek-ai/deepseek-coder-1.3b-base," char c = string.charAt(i); -",155,1.0 +",155,1.0 2025-09-22T13:59:14.488086,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30434,7 +30434,7 @@ class Problem { } }",154.9765000017942,,Completions generated successfully. Ready for display in your IDE.,501fcdda-9a62-48b0-b0b4-f0ffc929549d,1,deepseek-ai/deepseek-coder-1.3b-base," v *= j; -",105,1.0 +",105,1.0 2025-09-22T13:59:14.574689,[1],"from typing import Tuple def even_odd_palindrome(n: int) -> Tuple[int, int]: @@ -30463,7 +30463,7 @@ def even_odd_palindrome(n: int) -> Tuple[int, int]: return ( sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 == 0), sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 != 0)",,85.80870000150753,,Completions generated successfully. Ready for display in your IDE.,e7b93de3-b5ce-4dc2-9780-cacbf09cf985,1,deepseek-ai/deepseek-coder-1.3b-base,", - )",62,0.8622 + )",62,0.8622 2025-09-22T13:59:14.653072,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -30492,7 +30492,7 @@ function is_equal_to_sum_even(n){ } }",78.3785000021453,,Completions generated successfully. Ready for display in your IDE.,300e0e63-48f0-4bfc-a339-5a8d7e430589,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T13:59:14.771963,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30539,7 +30539,7 @@ class Problem { } return count;",},118.89059999521123,,Completions generated successfully. Ready for display in your IDE.,d6c6e3b7-1704-40fa-a147-3e27fe039165,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:15.180581,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30579,7 +30579,7 @@ class Problem { } }",408.4459999940009,,Completions generated successfully. Ready for display in your IDE.,4da389d2-32a2-4402-aece-9dc4c02029a4,1,deepseek-ai/deepseek-coder-1.3b-base," int compareResult = wordsToUniqueCharacters.get(o1).compareTo(wordsToUniqueCharacters.get(o2)); -",378,0.9857 +",378,0.9857 2025-09-22T13:59:15.324464,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30617,7 +30617,7 @@ class Problem { } }",143.86260000173934,,Completions generated successfully. Ready for display in your IDE.,ab2e7d4f-8bc7-4d28-8ea2-9b36302a5b55,1,deepseek-ai/deepseek-coder-1.3b-base," return lst2; -",117,1.0 +",117,1.0 2025-09-22T13:59:15.439266,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -30635,7 +30635,7 @@ function choose_num(x, y){ return num; } ",114.7722999958205,,Completions generated successfully. Ready for display in your IDE.,3fe2f851-cfe3-4dad-8b47-cc159a5a3c51,1,deepseek-ai/deepseek-coder-1.3b-base," var num = 0; -",87,1.0 +",87,1.0 2025-09-22T13:59:15.570456,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -30671,7 +30671,7 @@ function choose_num(x, y){ ",130.5893999960972,,Completions generated successfully. Ready for display in your IDE.,f81a1983-6d35-425d-9819-5594766c07b0,1,deepseek-ai/deepseek-coder-1.3b-base," 1: 'i', -",105,0.9159 +",105,0.9159 2025-09-22T13:59:15.696591,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30703,7 +30703,7 @@ class Problem { } }",125.904200001969,,Completions generated successfully. Ready for display in your IDE.,1068b0f6-d6af-4d80-90cb-300cfb3c9f4e,1,deepseek-ai/deepseek-coder-1.3b-base," int left = 0; -",102,1.0 +",102,1.0 2025-09-22T13:59:15.793770,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30748,7 +30748,7 @@ class Problem { } return true;",},96.80040000239387,,Completions generated successfully. Ready for display in your IDE.,b3f4ad7b-d077-423a-b9e9-d2ad5a82a8fe,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:15.944315,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -30775,7 +30775,7 @@ def find_max(words: List[str]) -> str: return max_word ",150.68959999916842,,Completions generated successfully. Ready for display in your IDE.,a86d7ebb-6e2c-48bc-944c-29ffedb85f43,1,deepseek-ai/deepseek-coder-1.3b-base," if word < max_word: -",124,1.0 +",124,1.0 2025-09-22T13:59:16.039437,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30811,7 +30811,7 @@ class Problem { } }",94.52310000051511,,Completions generated successfully. Ready for display in your IDE.,8f1b643a-659c-4cc6-8b37-0d0f48289360,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:16.157702,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30842,7 +30842,7 @@ class Problem { } }",118.4685999978683,,Completions generated successfully. Ready for display in your IDE.,9964c897-1407-4f99-a7cc-0c8103535281,1,deepseek-ai/deepseek-coder-1.3b-base," inc = false; -",95,1.0 +",95,1.0 2025-09-22T13:59:16.297410,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -30857,7 +30857,7 @@ function filter_integers(values){ } ",139.72389999980805,,Completions generated successfully. Ready for display in your IDE.,d6049fd0-b706-4416-8bed-e046d0d1cf16,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(value); -",118,1.0 +",118,1.0 2025-09-22T13:59:16.427611,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -30885,7 +30885,7 @@ function pairs_sum_to_zero(l){ } ",129.76539999363013,,Completions generated successfully. Ready for display in your IDE.,7051dc52-0051-4e50-bd45-343341a39f86,1,deepseek-ai/deepseek-coder-1.3b-base," i = i + 1 -",104,1.0 +",104,1.0 2025-09-22T13:59:16.570713,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -30923,7 +30923,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",142.86020000145072,,Completions generated successfully. Ready for display in your IDE.,0a333ea4-632d-4ca7-8eaa-409d949f2b31,1,deepseek-ai/deepseek-coder-1.3b-base," return (y == x); -",119,0.8687 +",119,0.8687 2025-09-22T13:59:16.657091,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30963,7 +30963,7 @@ class Problem { } }",86.6939999978058,,Completions generated successfully. Ready for display in your IDE.,6aae5e67-5045-4c3e-b754-2122c4c9ca0f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:16.815204,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -30991,7 +30991,7 @@ function solve(s){ } ",157.0584000000963,,Completions generated successfully. Ready for display in your IDE.,a1f3d4d8-6e5c-42f3-8d87-7397b73429c4,1,deepseek-ai/deepseek-coder-1.3b-base," : char.toLowerCase(); -",124,1.0 +",124,1.0 2025-09-22T13:59:16.962370,[1],"def fizz_buzz(n: int) -> int: """"""Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. >>> fizz_buzz(50) @@ -31006,7 +31006,7 @@ function solve(s){ return x ",146.7720000000554,,Completions generated successfully. Ready for display in your IDE.,e9b4a184-6198-4721-b96b-2e299d82c453,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(n): -",123,1.0 +",123,1.0 2025-09-22T13:59:17.088399,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -31031,7 +31031,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",126.1681000032695,,Completions generated successfully. Ready for display in your IDE.,cafb5833-5744-4f85-b4f4-58c4be7053e0,1,deepseek-ai/deepseek-coder-1.3b-base," open_count = 0 -",104,1.0 +",104,1.0 2025-09-22T13:59:17.153651,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31063,7 +31063,7 @@ class Problem { } return sb.reverse().toString(); }",,64.98899999860441,,Completions generated successfully. Ready for display in your IDE.,aee0d838-aead-452b-b3d5-7ae2d8bd4809,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:17.258739,[1],"//Given an array of non-negative integers, return a cojs of the given array after sorting, // you will sort the given array in ascending order if the sum( first index value, last index value) is odd, // or sort it in descending order if the sum( first index value, last index value) is even. @@ -31087,7 +31087,7 @@ function sort_array(array){ } return newArr; }",104.98489999736194,,Completions generated successfully. Ready for display in your IDE.,b9af97a5-5b08-422e-acd7-b1a60dfad941,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here -",82,0.9177 +",82,0.9177 2025-09-22T13:59:17.518641,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -31142,7 +31142,7 @@ function by_length(arr){ return arr3; }",259.53600000502774,,Completions generated successfully. Ready for display in your IDE.,5f8c55a7-4a9e-4c67-8a6f-48a4da4b4f16,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i 0l) { -",184,0.9384 +",184,0.9384 2025-09-22T13:59:17.842939,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -31261,7 +31261,7 @@ function minPath(grid, k){ return path; }",112.24770000262652,,Completions generated successfully. Ready for display in your IDE.,ddafaca3-ebb2-4c79-a5c6-e999ff4b13ec,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:18.247599,[1],"//Create a function encrypt that takes a string as an argument and // returns a string encrypted with the alphabet being rotated. // The alphabet should be rotated in a manner such that the letters @@ -31286,7 +31286,7 @@ function encrypt(s){ ",405.04840000357945,,Completions generated successfully. Ready for display in your IDE.,efc7b107-a18b-4481-9bd7-0418493f1eb1,1,deepseek-ai/deepseek-coder-1.3b-base," //get index of letter from inputted word, then add it to our shifted value var index = (alphabet.indexOf)(s[i]); -",382,0.9175 +",382,0.9175 2025-09-22T13:59:18.839108,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -31317,7 +31317,7 @@ def unique_digits(x: List[int]) -> List[int]: for i in range(len(s)): for j in range(len(x)-1): if (s[i]==x[j+1]-x[j]): - ",567,0.9635 + ",567,0.9635 2025-09-22T13:59:18.902784,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -31350,7 +31350,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.91090000397526,,Completions generated successfully. Ready for display in your IDE.,529315d1-c2c5-43ea-a6da-32f89009994b,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,0.9621 +",38,0.9621 2025-09-22T13:59:18.957842,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31376,7 +31376,7 @@ class Problem { } } }",54.521899997780565,,Completions generated successfully. Ready for display in your IDE.,4a9cda01-323b-4ce8-b46f-c2eadfdae6cb,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,0.8716 +",31,0.8716 2025-09-22T13:59:19.063386,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31399,7 +31399,7 @@ class Problem { }"," } }",105.97119999874849,,Completions generated successfully. Ready for display in your IDE.,0554ca73-da1e-400a-aa41-8e221d134d70,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",82,0.9796 +",82,0.9796 2025-09-22T13:59:19.349619,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -31449,7 +31449,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",285.2721999952337,,Completions generated successfully. Ready for display in your IDE.,320caf3c-ede8-4b60-abc0-60dd90eaf2d2,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i=5; ((i * i) <= number); i+= 6){ -",260,0.9263 +",260,0.9263 2025-09-22T13:59:19.536803,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -31483,7 +31483,7 @@ function hex_key(num){ continue; } } -",161,1.0 +",161,1.0 2025-09-22T13:59:19.732847,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -31505,7 +31505,7 @@ function fizz_buzz(n){ } ",195.75560000521364,,Completions generated successfully. Ready for display in your IDE.,c0b9927a-5b0d-4e6d-838d-de07e0bf0388,1,deepseek-ai/deepseek-coder-1.3b-base," if (arr[j] === '7'){ -",173,1.0 +",173,1.0 2025-09-22T13:59:19.825118,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -31531,7 +31531,7 @@ function separate_paren_groups(paren_string){ } ",92.09130000090227,,Completions generated successfully. Ready for display in your IDE.,d714fe59-2da3-4a0f-a9c5-4817352765d8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9674 +",69,0.9674 2025-09-22T13:59:19.945344,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -31569,7 +31569,7 @@ function get_closest_vowel(word){ } }",119.49559999629855,,Completions generated successfully. Ready for display in your IDE.,e2664ff3-a366-4344-8b42-f7a85ddc7188,1,deepseek-ai/deepseek-coder-1.3b-base," for (;;) { -",95,0.8687 +",95,0.8687 2025-09-22T13:59:20.009187,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -31596,7 +31596,7 @@ function monotonic(l){ } } return true;",,64.1990000003716,,Completions generated successfully. Ready for display in your IDE.,a43d600b-b84f-414d-a745-c21fc9965f63,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:20.707546,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31624,7 +31624,7 @@ class Problem { return al; } - //",674,0.9264 + //",674,0.9264 2025-09-22T13:59:20.921133,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31673,7 +31673,7 @@ class Problem { } }",212.96030000667088,,Completions generated successfully. Ready for display in your IDE.,7dc873d4-8109-4129-b60d-44e91221eb38,1,deepseek-ai/deepseek-coder-1.3b-base," if (!hm.containsKey(sss)) { -",184,0.9762 +",184,0.9762 2025-09-22T13:59:21.028860,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -31699,7 +31699,7 @@ function is_multiply_prime(a){ } }",107.78170000412501,,Completions generated successfully. Ready for display in your IDE.,2db84e94-de80-486b-b8a0-46841c93775f,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",82,1.0 +",82,1.0 2025-09-22T13:59:21.146469,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -31738,7 +31738,7 @@ function valid_date(date){ } ",117.530399998941,,Completions generated successfully. Ready for display in your IDE.,88e52f67-ddf0-427f-b33e-640e133324eb,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",90,0.9124 +",90,0.9124 2025-09-22T13:59:21.359995,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -31768,7 +31768,7 @@ function parse_nested_parens(paren_string){ ",213.30110000417335,,Completions generated successfully. Ready for display in your IDE.,c4783c4e-2167-4d75-85e6-73b89346ad59,1,deepseek-ai/deepseek-coder-1.3b-base," if (paren_string[i] == ""(""){ -",189,1.0 +",189,1.0 2025-09-22T13:59:21.458211,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -31791,7 +31791,7 @@ function parse_nested_parens(paren_string){ return string ",97.61799999978393,,Completions generated successfully. Ready for display in your IDE.,ec858b00-f86b-4fd4-b39b-28af6451ef3b,1,deepseek-ai/deepseek-coder-1.3b-base," - prefix = string",73,1.0 + prefix = string",73,1.0 2025-09-22T13:59:21.598809,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31869,7 +31869,7 @@ class Problem { } }",140.72680000390392,,Completions generated successfully. Ready for display in your IDE.,3ecaa910-c323-4a6c-949c-2fc366518677,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 9; -",113,1.0 +",113,1.0 2025-09-22T13:59:21.843707,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31919,7 +31919,7 @@ class Problem { } }",244.59550000028685,,Completions generated successfully. Ready for display in your IDE.,69942065-7cd3-47d1-b3f9-c06387d1862a,1,deepseek-ai/deepseek-coder-1.3b-base," for (char c : wordInLowerCase.toCharArray()) { -",214,1.0 +",214,1.0 2025-09-22T13:59:22.035574,[1],"def sum_to_n(n: int) -> int: """"""sum_to_n is a function that sums numbers from 1 to n. >>> sum_to_n(30) @@ -31935,7 +31935,7 @@ class Problem { """""" "," ",191.45539999590255,,Completions generated successfully. Ready for display in your IDE.,2b16cf60-bafb-430a-b7af-cb0eae52c3d2,1,deepseek-ai/deepseek-coder-1.3b-base," return (n * (n + 1)) // 2 -",169,0.9677 +",169,0.9677 2025-09-22T13:59:22.102278,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31965,7 +31965,7 @@ class Problem { factors.add(n); return factors; }",,66.8218999999226,,Completions generated successfully. Ready for display in your IDE.,46645aa1-4d40-4a99-8958-205a34a4f272,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:22.210263,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -31991,7 +31991,7 @@ def smallest_change(arr: List[int]) -> int: return count ",107.24760000448441,,Completions generated successfully. Ready for display in your IDE.,c9c83a4c-5bab-4a4b-9ec9-8f65d238f695,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",83,1.0 +",83,1.0 2025-09-22T13:59:22.793620,[1],"//You will be given a string of words separated by commas or spaces. Your task is // to split the string into words and return an array of the words. // For example: @@ -32003,7 +32003,7 @@ function words_string(s){ "," return res; }",583.2049000018742,,Completions generated successfully. Ready for display in your IDE.,d93415af-1c00-4c4d-bcf3-7ecff1086563,1,deepseek-ai/deepseek-coder-1.3b-base," let arr = s.split(/[ ,]+/); //split() method splits up strings based on regex pattern passed as argument (in this case it'll match any whitespace) console.log('arr', arr); -",558,0.9285 +",558,0.9285 2025-09-22T13:59:23.050038,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32057,7 +32057,7 @@ class Problem { }",255.88620000053197,,Completions generated successfully. Ready for display in your IDE.,e4a63b34-6a7b-474a-bb2a-d34f7924e30d,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList newArr = new ArrayList<>(); -",215,0.993 +",215,0.993 2025-09-22T13:59:23.112190,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32088,7 +32088,7 @@ class Problem { return sum; } }",62.67680000019027,,Completions generated successfully. Ready for display in your IDE.,c9cc0aa8-92b9-4cad-8bf5-cec6ee86c9c1,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:59:23.165614,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -32117,7 +32117,7 @@ function sort_third(l){ } return x; }",52.64570000144886,,Completions generated successfully. Ready for display in your IDE.,95117b6b-61e4-4c5b-a703-eed822020a86,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:59:23.374012,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32154,7 +32154,7 @@ class Problem { return false; } }",208.55239999946207,,Completions generated successfully. Ready for display in your IDE.,557e2a25-31ce-4b54-a4a6-2d69683b26ae,1,deepseek-ai/deepseek-coder-1.3b-base," if (txt == null || txt.equals("""")) { -",171,0.9787 +",171,0.9787 2025-09-22T13:59:23.465892,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -32192,7 +32192,7 @@ function get_closest_vowel(word){ return result[0];"," } }",91.6367000027094,,Completions generated successfully. Ready for display in your IDE.,2cfefcb5-2fe2-47c1-afde-9c812c176ed3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:59:23.662070,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -32264,7 +32264,7 @@ function minPath(grid, k){ return path; }",195.75239999539917,,Completions generated successfully. Ready for display in your IDE.,890e62d8-3100-4dfa-8448-3549f2ce416c,1,deepseek-ai/deepseek-coder-1.3b-base," //find smallest number in all elements of matrix -",166,0.8289 +",166,0.8289 2025-09-22T13:59:24.107732,[1],"from typing import Tuple def reverse_delete(s: str, c: str) -> Tuple[str, bool]: @@ -32289,7 +32289,7 @@ def reverse_delete(s: str, c: str) -> Tuple[str, bool]: if new_str == new_str_reverse: return (new_str,True) else: - return (new_str,False)",413,0.9929 + return (new_str,False)",413,0.9929 2025-09-22T13:59:24.212490,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -32306,7 +32306,7 @@ function largest_prime_factor(n){ } return result; }",104.44480000296608,,Completions generated successfully. Ready for display in your IDE.,27932dd9-6a1a-411e-be03-ddbcbe684c19,1,deepseek-ai/deepseek-coder-1.3b-base," var result = 2; -",83,1.0 +",83,1.0 2025-09-22T13:59:24.393943,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -32337,7 +32337,7 @@ function total_match(lst1, lst2){ } ",181.4347999970778,,Completions generated successfully. Ready for display in your IDE.,8707f965-a996-4c83-9f5a-eaefc4f6e2b3,1,deepseek-ai/deepseek-coder-1.3b-base," sum_lst2 += item.length; -",156,1.0 +",156,1.0 2025-09-22T13:59:24.500148,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -32363,7 +32363,7 @@ function is_multiply_prime(a){ } }",106.32320000149775,,Completions generated successfully. Ready for display in your IDE.,cc7bc575-66ce-4ef1-b83f-983ed6fa9530,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",83,1.0 +",83,1.0 2025-09-22T13:59:24.596323,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32393,7 +32393,7 @@ class Problem { } }",95.36069999739993,,Completions generated successfully. Ready for display in your IDE.,23195242-d94e-4acf-829d-3d8f061ae9ea,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:24.689660,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -32428,7 +32428,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(-1)"," sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",93.05719999974826,,Completions generated successfully. Ready for display in your IDE.,c031527a-6a32-4688-b3a8-f61191ff35ba,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",70,1.0 +",70,1.0 2025-09-22T13:59:25.073203,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -32451,7 +32451,7 @@ function fizz_buzz(n){ ",383.5225000002538,,Completions generated successfully. Ready for display in your IDE.,a9233110-063c-4bc8-8f24-b4fcc1a39843,1,deepseek-ai/deepseek-coder-1.3b-base," // console.log(""i: "", i); if((i % 11 === 0 || i % 13 === 0)){ -",360,0.9577 +",360,0.9577 2025-09-22T13:59:25.342324,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32495,7 +32495,7 @@ class Problem { } }",269.2650999961188,,Completions generated successfully. Ready for display in your IDE.,88f738ae-5144-4ca5-83f0-307e18708961,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sortedList.size(); i++) { -",245,1.0 +",245,1.0 2025-09-22T13:59:25.594737,[1],"from typing import Tuple def reverse_delete(s: str, c: str) -> Tuple[str, bool]: @@ -32519,7 +32519,7 @@ def reverse_delete(s: str, c: str) -> Tuple[str, bool]: ",252.22609999764245,,Completions generated successfully. Ready for display in your IDE.,88e0a0dc-9426-4b67-863a-48251082c6e9,1,deepseek-ai/deepseek-coder-1.3b-base," new_str_reverse = new_str[::-1] -",226,0.9964 +",226,0.9964 2025-09-22T13:59:25.754500,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -32577,7 +32577,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.65739999955986,,Completions generated successfully. Ready for display in your IDE.,c46779f8-6d6d-4447-a559-fc12a420c9a4,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 2.7: -",131,1.0 +",131,1.0 2025-09-22T13:59:25.968744,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32639,7 +32639,7 @@ class Problem { } }",213.941199996043,,Completions generated successfully. Ready for display in your IDE.,934e4ce0-b516-47b1-8cae-8c11e4d4a8f7,1,deepseek-ai/deepseek-coder-1.3b-base," if ((number % l) == 0) { -",184,1.0 +",184,1.0 2025-09-22T13:59:26.250245,[1],"//Given a string 'text', return its md5 hash equivalent string. // If 'text' is an empty string, return undefined. // >>> string_to_md5(""Hello world"") @@ -32652,7 +32652,7 @@ function string_to_md5(text){ }else { console.log (stringToMd5(text)); - ",258,0.8938 + ",258,0.8938 2025-09-22T13:59:26.438768,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -32719,7 +32719,7 @@ function numerical_letter_grade(grades){ return grades_array; }",188.66089999937685,,Completions generated successfully. Ready for display in your IDE.,b163ba02-0d91-4aaf-a040-9ae9886e9883,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""B+""); -",160,1.0 +",160,1.0 2025-09-22T13:59:26.596776,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32751,7 +32751,7 @@ class Problem { } }",157.78950000094483,,Completions generated successfully. Ready for display in your IDE.,25f652d9-4285-4bbf-b4db-20e773fb194b,1,deepseek-ai/deepseek-coder-1.3b-base," b1 = -b1; -",131,1.0 +",131,1.0 2025-09-22T13:59:26.683533,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32774,7 +32774,7 @@ class Problem { public static boolean primeLength(String string) { return new java.math.BigInteger(String.valueOf(string.length())).isProbablePrime(100);",},86.36439999827417,,Completions generated successfully. Ready for display in your IDE.,2dfd405f-c3e7-4785-9bf4-70240716f7fd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:26.839744,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -32829,7 +32829,7 @@ function by_length(arr){ return arr3; }",155.8835999967414,,Completions generated successfully. Ready for display in your IDE.,b19d5f22-8c39-45c5-b40a-f7df5cf88fc8,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Two""); -",131,1.0 +",131,1.0 2025-09-22T13:59:27.083734,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32877,7 +32877,7 @@ class Problem { } }",243.24760000308743,,Completions generated successfully. Ready for display in your IDE.,4d0ea582-65ad-4abc-bf1a-663875d323fb,1,deepseek-ai/deepseek-coder-1.3b-base," int indexPlanet2 = planets.indexOf(planet2); -",212,1.0 +",212,1.0 2025-09-22T13:59:27.162490,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32917,7 +32917,7 @@ class Problem { return res; } }",78.99929999985034,,Completions generated successfully. Ready for display in your IDE.,f4de7919-a828-4856-922b-1849203649e6,1,deepseek-ai/deepseek-coder-1.3b-base," - ",51,1.0 + ",51,1.0 2025-09-22T13:59:27.255596,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32948,7 +32948,7 @@ class Problem { } }",92.62149999995017,,Completions generated successfully. Ready for display in your IDE.,4c7374a5-c6fd-4a1d-9ca0-4e4645eba50a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:27.346609,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -32972,7 +32972,7 @@ function sort_numbers(numbers){ numString = numString.join("" ""); return numString; }",91.45210000133375,,Completions generated successfully. Ready for display in your IDE.,96bf54db-0e62-499f-b1e4-175772549c15,1,deepseek-ai/deepseek-coder-1.3b-base,"}; -",49,0.9535 +",49,0.9535 2025-09-22T13:59:27.547825,[1],"//For a given array of input numbers, calculate Mean Absolute Deviation // around the mean of this dataset. // Mean Absolute Deviation is the average absolute difference between each @@ -32992,7 +32992,7 @@ function mean_absolute_deviation(numbers){ } ",200.384199997643,,Completions generated successfully. Ready for display in your IDE.,e460d226-c44e-41b3-850a-bdb0e04ab959,1,deepseek-ai/deepseek-coder-1.3b-base," sum+=Math.abs(numbers[i]); -",176,0.9433 +",176,0.9433 2025-09-22T13:59:27.709938,[1],"from typing import List def eat(number: int, need: int, remaining: int) -> List[int]: @@ -33029,7 +33029,7 @@ def eat(number: int, need: int, remaining: int) -> List[int]: Have fun :) """""" ",,162.61060000397265,,Completions generated successfully. Ready for display in your IDE.,973bbdf9-0250-4854-ab7d-b0b0abdcabae,1,deepseek-ai/deepseek-coder-1.3b-base," # TODO: Write your code here - pass",136,0.9293 + pass",136,0.9293 2025-09-22T13:59:27.946571,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33091,7 +33091,7 @@ class Problem { } }",236.19870000402443,,Completions generated successfully. Ready for display in your IDE.,cfe57f97-2f27-4c63-b147-07f3901b6506,1,deepseek-ai/deepseek-coder-1.3b-base," else if (item == 2 || item == 3) { -",205,0.987 +",205,0.987 2025-09-22T13:59:28.198118,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33131,7 +33131,7 @@ class Problem { } }",251.17599999794038,,Completions generated successfully. Ready for display in your IDE.,9bbea9fb-f557-46b1-8bf0-ea80ae2579cc,1,deepseek-ai/deepseek-coder-1.3b-base," Collections.sort(words, new Comparator() { -",223,1.0 +",223,1.0 2025-09-22T13:59:28.314902,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -33208,7 +33208,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",116.32470000040485,,Completions generated successfully. Ready for display in your IDE.,35600dc2-f6fa-42db-bb08-4a74a0fb0df4,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:59:28.424921,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -33231,7 +33231,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return count ",109.5770999963861,,Completions generated successfully. Ready for display in your IDE.,f32cd09d-8ea9-43bb-bb45-d72303a486b6,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",83,1.0 +",83,1.0 2025-09-22T13:59:28.602958,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33276,7 +33276,7 @@ class Problem { } }",178.38009999832138,,Completions generated successfully. Ready for display in your IDE.,6bec187f-ce51-40a6-bfe9-d447bb4009be,1,deepseek-ai/deepseek-coder-1.3b-base," for (int prime : primes) { -",152,1.0 +",152,1.0 2025-09-22T13:59:28.771776,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33326,7 +33326,7 @@ class Problem { return ret; } }",168.37710000254447,,Completions generated successfully. Ready for display in your IDE.,e0ee3f03-236e-4ad2-b2c9-b22b4aa1272b,1,deepseek-ai/deepseek-coder-1.3b-base," var ret = new ArrayList(); -",141,0.9387 +",141,0.9387 2025-09-22T13:59:28.922485,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -33384,7 +33384,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",150.23870000004536,,Completions generated successfully. Ready for display in your IDE.,0b10560e-cc1c-4afd-bc6e-8e89a02ccf6f,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('A') -",121,1.0 +",121,1.0 2025-09-22T13:59:28.989577,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -33411,7 +33411,7 @@ function triples_sum_to_zero(l){ } } return false;",,67.38290000066627,,Completions generated successfully. Ready for display in your IDE.,dd08ad3e-35c2-4ce1-b372-cb4e7948729e,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:29.146939,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -33466,7 +33466,7 @@ function by_length(arr){ return arr3; }",156.6878000012366,,Completions generated successfully. Ready for display in your IDE.,23dd7611-2e0f-4867-b261-bd5d1c159522,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""One""); -",131,1.0 +",131,1.0 2025-09-22T13:59:29.852821,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -33497,7 +33497,7 @@ function is_happy(s){ //check each letter in sequence with next two, if they're same then it isn't happy for(let i=0; i bool: """"""Your task is to write a function that returns true if a number x is a simple power of n and false in other cases. @@ -33551,7 +33551,7 @@ class Problem { elif x == 1: return True else: -",186,0.9583 +",186,0.9583 2025-09-22T13:59:30.600278,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -33597,7 +33597,7 @@ function histogram(test){ //finding highest value in dictionary to determine which keys are equal for (var key in count) { -",315,0.8125 +",315,0.8125 2025-09-22T13:59:30.750276,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -33655,7 +33655,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",149.74119999533286,,Completions generated successfully. Ready for display in your IDE.,5b3d84eb-91f7-4be9-8425-4804d83bc265,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('B') -",121,1.0 +",121,1.0 2025-09-22T13:59:30.855286,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -33681,7 +33681,7 @@ function is_multiply_prime(a){ }else{"," } }",104.33029999694554,,Completions generated successfully. Ready for display in your IDE.,3e3a739b-902c-4340-9af6-2aa8125777c3,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:59:30.959283,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33730,7 +33730,7 @@ class Problem { } return valid_date;",},104.48460000043269,,Completions generated successfully. Ready for display in your IDE.,f4d8da4e-f306-452d-b3ae-fb9125a73109,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:59:31.069284,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33773,7 +33773,7 @@ class Problem { } }",109.16569999972126,,Completions generated successfully. Ready for display in your IDE.,8b55fb2c-eb78-4424-84d9-064baf06596a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",85,1.0 +",85,1.0 2025-09-22T13:59:31.772570,[1],"//You will be given a number in decimal form and your task is to convert it to // binary format. The function should return a string, with each character representing a binary // number. Each character in the string will be '0' or '1'. @@ -33789,7 +33789,7 @@ function decimal_to_binary(decimal){ ",703.0773000005865,,Completions generated successfully. Ready for display in your IDE.,d096d4e6-e07f-4018-a98c-b1560487791c,1,deepseek-ai/deepseek-coder-1.3b-base," let binary = """"; //create empty variable for binary while (decimal > 0 ){ //while loop that runs until decimal reaches zero if ((decimal % 2) === 0){ //if statement that checks remainder of decimal divided by two - binary += ""0""; ",676,0.9297 + binary += ""0""; ",676,0.9297 2025-09-22T13:59:31.842543,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -33813,7 +33813,7 @@ function decimal_to_binary(decimal){ else:"," pass ",70.07029999658698,,Completions generated successfully. Ready for display in your IDE.,633f43a5-f92a-49c5-acfb-665b9c664bf4,1,deepseek-ai/deepseek-coder-1.3b-base," - ",47,0.7477 + ",47,0.7477 2025-09-22T13:59:31.951548,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33857,7 +33857,7 @@ class Problem { }"," } }",106.98679999768501,,Completions generated successfully. Ready for display in your IDE.,9ef4da64-a815-4277-9c0a-c11e62abb792,1,deepseek-ai/deepseek-coder-1.3b-base," return """"; -",79,1.0 +",79,1.0 2025-09-22T13:59:32.025546,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -33895,7 +33895,7 @@ function even_odd_palindrome(n){ } } return [even, odd];",,73.88690000516362,,Completions generated successfully. Ready for display in your IDE.,d7d6d331-8c07-4339-9044-d37de990dbca,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:59:32.114543,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33915,7 +33915,7 @@ class Problem { } }",88.12229999603005,,Completions generated successfully. Ready for display in your IDE.,c5e70b33-cd74-47b1-8838-f6a660b6ece4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:32.220542,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -33932,7 +33932,7 @@ function has_close_elements(numbers, threshold){ } ",106.45049999584444,,Completions generated successfully. Ready for display in your IDE.,9d07738c-11b9-442d-a090-8cfb119d777c,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:59:32.390547,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33954,7 +33954,7 @@ class Problem { } }",169.4777999946382,,Completions generated successfully. Ready for display in your IDE.,b9ad06b1-24ee-42fb-81dd-a9e0c86460a7,1,deepseek-ai/deepseek-coder-1.3b-base," String s = Long.toString(i); -",145,1.0 +",145,1.0 2025-09-22T13:59:32.620156,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -33984,7 +33984,7 @@ def count_nums(arr: List[int]) -> int: return n ",228.95080000307644,,Completions generated successfully. Ready for display in your IDE.,170878ca-7695-46d0-8fc8-4a0b56d0cd0f,1,deepseek-ai/deepseek-coder-1.3b-base," if s > 0: -",104,1.0 +",104,1.0 2025-09-22T13:59:32.696708,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -34017,7 +34017,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",76.87949999672128,,Completions generated successfully. Ready for display in your IDE.,dd97bc89-0660-4666-993c-33d749f2f4b6,1,deepseek-ai/deepseek-coder-1.3b-base,"2 -",49,1.0 +",49,1.0 2025-09-22T13:59:32.909839,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34066,7 +34066,7 @@ class Problem { } }",212.5746999954572,,Completions generated successfully. Ready for display in your IDE.,48e803e4-9618-401a-a930-9d1c30695ec6,1,deepseek-ai/deepseek-coder-1.3b-base," for (String key : hm.keySet()) { -",184,1.0 +",184,1.0 2025-09-22T13:59:33.017935,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -34094,7 +34094,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: if any(not x.isupper() for x in dict.keys()):"," return True ",108.23600000003353,,Completions generated successfully. Ready for display in your IDE.,b2eaba83-2f4f-49d4-81d2-19f258c83258,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",74,1.0 +",74,1.0 2025-09-22T13:59:33.122405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34142,7 +34142,7 @@ class Problem { } }",104.1830000031041,,Completions generated successfully. Ready for display in your IDE.,ef43db08-67b3-4c01-834a-85bc08e63944,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:59:33.165948,[1],"def fib(n: int) -> int: """"""Return n-th Fibonacci number. >>> fib(10) @@ -34157,7 +34157,7 @@ class Problem { f0, f1 = f1, f0 + f1 return f1 -",42.75439999764785,,Completions generated successfully. Ready for display in your IDE.,b180014b-5014-4e94-9266-0ff14b97e49e,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",42.75439999764785,,Completions generated successfully. Ready for display in your IDE.,b180014b-5014-4e94-9266-0ff14b97e49e,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:33.231950,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -34171,7 +34171,7 @@ function rescale_to_unit(numbers){ unit.push((numbers[i] - min) / sub); } return unit;",,65.55200000002515,,Completions generated successfully. Ready for display in your IDE.,6cbed3bc-7b52-4a67-a9a9-14eac6624a96,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:33.299947,[1],"//You are given an array of numbers. // You need to return the sum of squared numbers in the given array, // round each element in the array to the upper int(Ceiling) first. @@ -34188,7 +34188,7 @@ function rescale_to_unit(numbers){ // 6 function sum_squares(lst){ return lst.map(num => Math.ceil(num)).reduce((sum, num) => sum + num ** 2, 0);",,68.1966000047396,,Completions generated successfully. Ready for display in your IDE.,58ebb9bc-06ad-4086-8f5e-ecb801ca2483,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:33.478948,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34266,7 +34266,7 @@ class Problem { } }",178.08769999828655,,Completions generated successfully. Ready for display in your IDE.,efedbc4d-be71-48b2-a267-9312d58ef3fd,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 1000; -",147,1.0 +",147,1.0 2025-09-22T13:59:33.805951,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34286,7 +34286,7 @@ class Problem { } }",327.19250000081956,,Completions generated successfully. Ready for display in your IDE.,d0aaf51f-0bc4-4a33-ad50-cca7a8f5d749,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < Math.min(a.length(),b.length()); ++i){ -",303,1.0 +",303,1.0 2025-09-22T13:59:33.883949,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -34313,7 +34313,7 @@ function next_smallest(lst){ return second_smallest; }",77.91079999879003,,Completions generated successfully. Ready for display in your IDE.,7249b2ee-b3d6-4792-85b9-45556b53181a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",52,1.0 +",52,1.0 2025-09-22T13:59:33.946952,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34340,7 +34340,7 @@ class Problem { return p; } }",61.973100004252046,,Completions generated successfully. Ready for display in your IDE.,4fcb13d6-21a6-4d27-8c29-f3879700756f,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T13:59:34.019951,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -34358,7 +34358,7 @@ function x_or_y(n, x, y){ return x; }",73.12719999754336,,Completions generated successfully. Ready for display in your IDE.,2c56c849-9e02-496a-b908-0290bbbc9ccc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",51,1.0 +",51,1.0 2025-09-22T13:59:34.154949,[1],"from typing import List def get_odd_collatz(n: int) -> List[int]: @@ -34392,7 +34392,7 @@ def get_odd_collatz(n: int) -> List[int]: ",135.11060000018915,,Completions generated successfully. Ready for display in your IDE.,85cf0d2c-cd28-4161-bbfb-89f458b1597c,1,deepseek-ai/deepseek-coder-1.3b-base," current_num = n -",111,1.0 +",111,1.0 2025-09-22T13:59:34.298951,[1],"from typing import Union, List def split_words(txt: str) -> Union[List[str], int]: @@ -34415,7 +34415,7 @@ def split_words(txt: str) -> Union[List[str], int]: return len([letter for letter in txt if letter in alphabet and alphabet.index(letter) % 2 == 1]) ",143.60669999587117,,Completions generated successfully. Ready for display in your IDE.,4ce22756-c2fd-4cb2-bf7f-4b8da219a9ba,1,deepseek-ai/deepseek-coder-1.3b-base," if ' ' in txt: -",114,0.9796 +",114,0.9796 2025-09-22T13:59:34.474198,[1],"def largest_divisor(n: int) -> int: """""" For a given number n, find the largest number that divides n evenly, smaller than n >>> largest_divisor(15) @@ -34425,7 +34425,7 @@ def split_words(txt: str) -> Union[List[str], int]: return divisor ",174.7255000009318,,Completions generated successfully. Ready for display in your IDE.,e67739b0-d775-403d-b890-9caa3454c7b1,1,deepseek-ai/deepseek-coder-1.3b-base," while not (n % divisor == 0): -",153,0.9946 +",153,0.9946 2025-09-22T13:59:34.566198,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34452,7 +34452,7 @@ class Problem { return Problem.fib((n - 1)) + Problem.fib((n - 2));"," } }",92.13820000150008,,Completions generated successfully. Ready for display in your IDE.,4df20da9-7c94-494d-ae85-f7ddaf04ee2d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:34.692632,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34486,7 +34486,7 @@ class Problem { } }",126.1305000007269,,Completions generated successfully. Ready for display in your IDE.,f12ec6d8-eada-4306-b0fe-c7a4e67c169a,1,deepseek-ai/deepseek-coder-1.3b-base," int count = 0; -",103,1.0 +",103,1.0 2025-09-22T13:59:34.843311,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -34520,7 +34520,7 @@ function count_nums(arr){ return count; }",149.55429999827174,,Completions generated successfully. Ready for display in your IDE.,c8537621-1f06-4b82-9685-6f4268829224,1,deepseek-ai/deepseek-coder-1.3b-base," if (Number(str)) { -",125,0.9647 +",125,0.9647 2025-09-22T13:59:34.930312,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -34543,7 +34543,7 @@ function compare(game, guess){ } }","} ",87.17099999921629,,Completions generated successfully. Ready for display in your IDE.,1629f8ce-b2c3-465c-83ad-2680be80a5e4,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",64,0.9863 +",64,0.9863 2025-09-22T13:59:34.995312,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34566,7 +34566,7 @@ class Problem { } return max; }",,64.26539999665692,,Completions generated successfully. Ready for display in your IDE.,0c7810de-dd1e-4e95-b8dc-b6d8697f50ee,1,deepseek-ai/deepseek-coder-1.3b-base," -}",40,1.0 +}",40,1.0 2025-09-22T13:59:35.305316,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34583,7 +34583,7 @@ class Problem { public static String concatenate(ArrayList strings) { "," } }",310.4987999977311,,Completions generated successfully. Ready for display in your IDE.,cf86393d-3498-4d46-99e0-af2a0f077bc1,1,deepseek-ai/deepseek-coder-1.3b-base," return strings.stream().reduce("""", (s1, s2) -> s1 + s2); -",288,0.9869 +",288,0.9869 2025-09-22T13:59:35.390324,[1],"//Given a positive integer N, return the total sum of its digits in binary. // Example // >>> solve(1000) @@ -34605,7 +34605,7 @@ function solve(N){ total += parseInt(digit);"," return total.toString(2); }",84.77750000020023,,Completions generated successfully. Ready for display in your IDE.,766927a3-a730-4a51-bfce-53c5b9fc9729,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:35.553310,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -34626,7 +34626,7 @@ function solve(N){ ",161.92700000101468,,Completions generated successfully. Ready for display in your IDE.,916a50e2-2415-41db-a2c1-e01ca9eb728b,1,deepseek-ai/deepseek-coder-1.3b-base," return round(area, 2) -",134,1.0 +",134,1.0 2025-09-22T13:59:35.637312,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -34652,7 +34652,7 @@ function fibfib(n){ ",84.17890000418993,,Completions generated successfully. Ready for display in your IDE.,406b5588-0071-46b3-a687-1b8a5ced94a7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T13:59:35.712313,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34673,7 +34673,7 @@ class Problem { .collect(Collectors.toSet()) ); }",,74.36109999980545,,Completions generated successfully. Ready for display in your IDE.,bfa623c0-802d-4cf6-b3c5-9a8fc55735ed,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:59:36.381313,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -34723,7 +34723,7 @@ function int_to_mini_roman(number){ function mini_roman(){ //your code here... - ",644,0.966 + ",644,0.966 2025-09-22T13:59:36.580312,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34774,7 +34774,7 @@ class Problem { } }",199.5810000007623,,Completions generated successfully. Ready for display in your IDE.,99f5499f-d7fa-4156-af23-162b79aa5155,1,deepseek-ai/deepseek-coder-1.3b-base," if (tokens.length != 2) { -",172,1.0 +",172,1.0 2025-09-22T13:59:36.961181,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34803,7 +34803,7 @@ class Problem { }",379.5810000010533,,Completions generated successfully. Ready for display in your IDE.,873a88cb-3cdd-4bdd-b99a-e87660181672,1,deepseek-ai/deepseek-coder-1.3b-base," if ((arr.get(i).longValue()) >= arr.get(i-1).longValue()) continue; else { -",353,0.9519 +",353,0.9519 2025-09-22T13:59:37.099882,[1],"def car_race_collision(n: int) -> int: """""" Imagine a road that's a perfectly straight infinitely long line. @@ -34824,7 +34824,7 @@ class Problem { count += 1 return count",138.65619999705814,,Completions generated successfully. Ready for display in your IDE.,2fdbecb4-ed73-4057-9ddc-1559d8f6b617,1,deepseek-ai/deepseek-coder-1.3b-base," // 2 count = 0 -",114,1.0 +",114,1.0 2025-09-22T13:59:37.194542,[1],"def is_equal_to_sum_even(n: int) -> bool: """"""Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers Example @@ -34839,7 +34839,7 @@ class Problem { ",94.86179999657907,,Completions generated successfully. Ready for display in your IDE.,513e8522-b0d8-4763-bc69-bffcd634cf8d,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",73,1.0 +",73,1.0 2025-09-22T13:59:37.395754,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -34854,7 +34854,7 @@ function rescale_to_unit(numbers){ return unit; } ",200.66339999902993,,Completions generated successfully. Ready for display in your IDE.,8b4e0838-4c45-482e-b2cf-042487dc0d71,1,deepseek-ai/deepseek-coder-1.3b-base," var min = Math.min.apply(Math, numbers); -",178,1.0 +",178,1.0 2025-09-22T13:59:37.491661,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -34909,7 +34909,7 @@ function by_length(arr){ return arr3; }",95.92669999983627,,Completions generated successfully. Ready for display in your IDE.,3edc1426-28a2-41b0-83af-fc65b8b60d01,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:37.623605,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -34942,7 +34942,7 @@ function digits(n){ ",131.79759999911766,,Completions generated successfully. Ready for display in your IDE.,cdeec168-0727-4cf8-adb6-81a2a8083742,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""Error""); -",111,0.9455 +",111,0.9455 2025-09-22T13:59:37.708336,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34970,7 +34970,7 @@ class Problem { public static boolean iscube(long a) { return a==0l||(a>0l&&Math.pow(Math.round(Math.pow(a, 1.0/3.0)), 3)==a)||(a<0l&&Math.pow(Math.round(Math.pow(-a, 1.0/3.0)), 3)==-a);",},84.38870000099996,,Completions generated successfully. Ready for display in your IDE.,497a753c-37fc-4d72-8620-ea71c5191754,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:59:37.801773,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35021,7 +35021,7 @@ class Problem { } return ""Yes"";",},93.52649999345886,,Completions generated successfully. Ready for display in your IDE.,6e17ee61-4c14-4922-bde6-902260dfe74d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:59:37.971334,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35068,7 +35068,7 @@ class Problem { } }",168.75260000233538,,Completions generated successfully. Ready for display in your IDE.,b2872310-f711-49fd-8165-19f45f6f5aa3,1,deepseek-ai/deepseek-coder-1.3b-base," long level = 0l; -",136,1.0 +",136,1.0 2025-09-22T13:59:38.108806,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -35098,7 +35098,7 @@ function parse_nested_parens(paren_string){ ",137.47250000596978,,Completions generated successfully. Ready for display in your IDE.,fc9a4ffc-f601-4066-844a-b2850981d127,1,deepseek-ai/deepseek-coder-1.3b-base," maxlevel = level; -",113,1.0 +",113,1.0 2025-09-22T13:59:38.305962,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35163,7 +35163,7 @@ class Problem { } }",196.46510000166018,,Completions generated successfully. Ready for display in your IDE.,b276478a-80c3-49bb-97ec-639ab06a05c1,1,deepseek-ai/deepseek-coder-1.3b-base," letterGrades.add(""C+""); -",164,1.0 +",164,1.0 2025-09-22T13:59:38.380438,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -35185,7 +35185,7 @@ function modp(n, p){ ",74.85900000028778,,Completions generated successfully. Ready for display in your IDE.,e567d1b3-1bcc-48d4-b5f2-813051a990e1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,1.0 +",53,1.0 2025-09-22T13:59:38.601242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35216,7 +35216,7 @@ class Problem { } }",219.96219999709865,,Completions generated successfully. Ready for display in your IDE.,e5539ace-8fd6-4448-b378-1047b9e0497f,1,deepseek-ai/deepseek-coder-1.3b-base," int res_num = x_num * n_num; -",194,1.0 +",194,1.0 2025-09-22T13:59:38.728823,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35266,7 +35266,7 @@ class Problem { } }",127.30390000069747,,Completions generated successfully. Ready for display in your IDE.,9818c6ca-36a6-4549-ba6f-0ff648f8cc73,1,deepseek-ai/deepseek-coder-1.3b-base," case 'i': -",100,1.0 +",100,1.0 2025-09-22T13:59:38.786317,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -35293,7 +35293,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return False return True ",56.84130000008736,,Completions generated successfully. Ready for display in your IDE.,8408e39e-0de0-49f3-af4b-a8da90f5f477,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,0.8678 +",32,0.8678 2025-09-22T13:59:38.875166,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -35317,7 +35317,7 @@ function vowels_count(s){ } ",88.98539999790955,,Completions generated successfully. Ready for display in your IDE.,3c89c1a0-c89d-4103-9f99-37a9ac69489a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",46,1.0 +",46,1.0 2025-09-22T13:59:39.122212,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35382,7 +35382,7 @@ class Problem { } }",247.43579999631038,,Completions generated successfully. Ready for display in your IDE.,3520d417-eca3-474f-963b-388836aeec1c,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 1.3f) { -",216,1.0 +",216,1.0 2025-09-22T13:59:39.330666,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -35411,7 +35411,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",207.66359999834094,,Completions generated successfully. Ready for display in your IDE.,3c506ad3-2425-4b6a-8267-c7725390f46b,1,deepseek-ai/deepseek-coder-1.3b-base," best_strength = 0 best_extension = '' -",181,0.9612 +",181,0.9612 2025-09-22T13:59:39.507006,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -35439,7 +35439,7 @@ function next_smallest(lst){ }",},175.83400000148686,,Completions generated successfully. Ready for display in your IDE.,3a88e09e-9464-4cd5-8c6b-22bd80d0380d,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ return second_smallest; } -",149,1.0 +",149,1.0 2025-09-22T13:59:39.665395,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -35482,7 +35482,7 @@ function get_odd_collatz(n){ } ",158.3165000047302,,Completions generated successfully. Ready for display in your IDE.,11a7b5c3-b22e-4db0-9a4f-70dec137a61a,1,deepseek-ai/deepseek-coder-1.3b-base," var odd_numbers = [] -",131,0.9292 +",131,0.9292 2025-09-22T13:59:39.934029,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -35501,7 +35501,7 @@ function count_upper(s){ return result; }",268.459899998561,,Completions generated successfully. Ready for display in your IDE.,65f6fe7a-9f12-4f98-a84c-d76415e58c83,1,deepseek-ai/deepseek-coder-1.3b-base," if ((vowel).indexOf(s[index]) !== -1) { -",238,0.9781 +",238,0.9781 2025-09-22T13:59:40.056990,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35546,7 +35546,7 @@ class Problem { } }",122.94340000516968,,Completions generated successfully. Ready for display in your IDE.,47d04767-50e1-497c-8ae9-71679bbf7d05,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",92,1.0 +",92,1.0 2025-09-22T13:59:40.153738,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -35585,7 +35585,7 @@ function exchange(lst1, lst2){ return ""NO"";","} ",95.6196999977692,,Completions generated successfully. Ready for display in your IDE.,f07abe3f-a62d-443e-b926-ab59340c7333,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:40.262742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35612,7 +35612,7 @@ class Problem { } }",108.63680000329623,,Completions generated successfully. Ready for display in your IDE.,4e6f90a6-ccaf-43a0-bdc7-7bfa69dd8919,1,deepseek-ai/deepseek-coder-1.3b-base," } else - ",83,0.9867 + ",83,0.9867 2025-09-22T13:59:40.369629,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -35637,7 +35637,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: groups.append(group)"," return groups ",107.15379999601282,,Completions generated successfully. Ready for display in your IDE.,34f68f71-bc92-400d-ac6c-fd52e777f21e,1,deepseek-ai/deepseek-coder-1.3b-base," group = '' -",84,1.0 +",84,1.0 2025-09-22T13:59:40.484522,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -35686,7 +35686,7 @@ function check_dict_case(dict){ return true; }",114.28909999813186,,Completions generated successfully. Ready for display in your IDE.,f0f152e5-2b37-482e-a605-ddb873f8f2c7,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",88,1.0 +",88,1.0 2025-09-22T13:59:40.671431,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35728,7 +35728,7 @@ class Problem { } }",187.06720000045607,,Completions generated successfully. Ready for display in your IDE.,52faabc8-451d-42c4-9992-48c27e2b7997,1,deepseek-ai/deepseek-coder-1.3b-base," String s3 = s1+s2; -",161,1.0 +",161,1.0 2025-09-22T13:59:40.741447,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -35777,7 +35777,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",69.77750000078231,,Completions generated successfully. Ready for display in your IDE.,2b6e7433-c657-4a5a-8514-3ffd518bf7a1,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:59:41.459703,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -35860,7 +35860,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i # return b # else: # return None - ",688,0.9495 + ",688,0.9495 2025-09-22T13:59:41.583162,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -35900,7 +35900,7 @@ function match_parens(lst){ return 'No'; }",122.98310000187485,,Completions generated successfully. Ready for display in your IDE.,ba6b7ba0-b536-4a7d-8f0f-361b8c70bf84,1,deepseek-ai/deepseek-coder-1.3b-base," return false; - ",99,0.9889 + ",99,0.9889 2025-09-22T13:59:41.997639,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -35930,7 +35930,7 @@ function split_words(txt){ }",414.26639999554027,,Completions generated successfully. Ready for display in your IDE.,cf178479-ba16-4f15-918c-4362c1a849cb,1,deepseek-ai/deepseek-coder-1.3b-base," const letter = txt[i]; if (letter === letter.toLowerCase() && /^[A-Z]/.test(letter)) { -",390,0.9372 +",390,0.9372 2025-09-22T13:59:42.210613,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -35957,7 +35957,7 @@ function split_words(txt){ ",211.9862000035937,,Completions generated successfully. Ready for display in your IDE.,7cab6ac0-58a6-49e1-8d41-670cc6958943,1,deepseek-ai/deepseek-coder-1.3b-base," if len(prime_factors) == 3: -",189,1.0 +",189,1.0 2025-09-22T13:59:42.328831,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35986,7 +35986,7 @@ class Problem { } }",118.40150000352878,,Completions generated successfully. Ready for display in your IDE.,f7001554-b9d5-4a62-acb4-37b481828d2f,1,deepseek-ai/deepseek-coder-1.3b-base," return sqrt; -",93,1.0 +",93,1.0 2025-09-22T13:59:42.848793,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -36003,7 +36003,7 @@ function rescale_to_unit(numbers){ if (!sub || !Number.isFinite(sub)) throw new Error('Invalid input'); //check whether there is any NaN or Infinity in this list var unit = []; -",492,0.907 +",492,0.907 2025-09-22T13:59:42.915221,[1],"//You're given an array of deposit and withdrawal operations on a bank account that starts with // zero balance. Your task is to detect if at any point the balance of account fallls below zero, and // at that point function should return true. Otherwise it should return false. @@ -36020,7 +36020,7 @@ function below_zero(operations){ } } return false;",,66.59870000294177,,Completions generated successfully. Ready for display in your IDE.,733ae7ab-dc80-4f21-8a86-2119b8d2a34a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:43.052208,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -36055,7 +36055,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",136.52830000501126,,Completions generated successfully. Ready for display in your IDE.,09b8eae0-513a-469b-9a82-338b59becaa2,1,deepseek-ai/deepseek-coder-1.3b-base," elif num < 0: -",104,1.0 +",104,1.0 2025-09-22T13:59:43.378987,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36089,7 +36089,7 @@ class Problem { if (current_depth > max_depth) { max_depth = current_depth; } -",301,1.0 +",301,1.0 2025-09-22T13:59:43.457238,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36119,7 +36119,7 @@ class Problem { return b.toString(); }",,77.79840000148397,,Completions generated successfully. Ready for display in your IDE.,54da171c-7f74-4cf7-830b-bf045311231b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,0.964 +",53,0.964 2025-09-22T13:59:44.152873,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36146,7 +36146,7 @@ class Problem { private static final Scanner scanner = new Scanner(System.in); public static void main(String[] args) throws IOException { - BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(System.",668,0.9934 + BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(System.",668,0.9934 2025-09-22T13:59:44.475261,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36195,7 +36195,7 @@ class Problem { } }",322.6725999993505,,Completions generated successfully. Ready for display in your IDE.,4065261e-7b09-4e9d-aadb-c49ae6e2d46a,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (month == 4 || month == 6 || month == 9 || month == 11) { -",290,1.0 +",290,1.0 2025-09-22T13:59:44.583906,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -36225,7 +36225,7 @@ function words_in_sentence(sentence){ }).join(' '); }",107.72510000242619,,Completions generated successfully. Ready for display in your IDE.,73d13699-cf19-46e6-95b8-c3b7ab157d78,1,deepseek-ai/deepseek-coder-1.3b-base," return null; -",82,0.9332 +",82,0.9332 2025-09-22T13:59:44.702797,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -36253,7 +36253,7 @@ function is_happy(s){ } return happy; }",118.96669999987353,,Completions generated successfully. Ready for display in your IDE.,068669be-fc1c-40bf-8ed7-c6e81fca647f,1,deepseek-ai/deepseek-coder-1.3b-base," let happy = true; -",93,0.9548 +",93,0.9548 2025-09-22T13:59:44.794849,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -36282,7 +36282,7 @@ function f(n){ return result; }",91.81580000586109,,Completions generated successfully. Ready for display in your IDE.,be43e98d-ac4f-47f1-8383-1511240bf85b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:59:44.963830,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -36354,7 +36354,7 @@ function minPath(grid, k){ return path; }",168.30789999949047,,Completions generated successfully. Ready for display in your IDE.,7e0d3bb5-5302-41d1-aed7-7e640d9b6775,1,deepseek-ai/deepseek-coder-1.3b-base," nextCell = curr; -",135,1.0 +",135,1.0 2025-09-22T13:59:45.278087,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -36426,7 +36426,7 @@ function minPath(grid, k){ return path; }",314.62740000279155,,Completions generated successfully. Ready for display in your IDE.,589b345e-5566-4685-8ecd-c514ecdd1b4c,1,deepseek-ai/deepseek-coder-1.3b-base," while(k-- !== 0 && currCell < 10 ** 9){ -",271,0.9379 +",271,0.9379 2025-09-22T13:59:45.389930,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -36468,7 +36468,7 @@ function compare_one(a, b){ } ",111.224599997513,,Completions generated successfully. Ready for display in your IDE.,4261dd7b-6f6f-4d18-9b3d-aaaa0d1bf3c8,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",84,1.0 +",84,1.0 2025-09-22T13:59:45.445644,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -36488,7 +36488,7 @@ function compare_one(a, b){ output_string += input_string[(input_string.index(i)+4)%26] return output_string ",54.936199994699564,,Completions generated successfully. Ready for display in your IDE.,1cb14596-07e2-4118-b973-d2ee5c54a004,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:45.584369,[1],"//Change numerical base of input number x to base. // return string representation after the conversion. // base numbers are less than 10. @@ -36504,7 +36504,7 @@ function change_base(x, base){ return change_base(q, base) + r.toString(); } ",138.3413000003202,,Completions generated successfully. Ready for display in your IDE.,d390fe9f-1b3a-4474-8169-1d1e0769fa0b,1,deepseek-ai/deepseek-coder-1.3b-base," const r = x % base; -",113,1.0 +",113,1.0 2025-09-22T13:59:45.724831,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -36534,7 +36534,7 @@ def count_nums(arr: List[int]) -> int: return n ",140.17320000129985,,Completions generated successfully. Ready for display in your IDE.,0964da28-b3c2-4d6b-b1b5-ec8d0f4d2f84,1,deepseek-ai/deepseek-coder-1.3b-base," s += int(j) -",113,1.0 +",113,1.0 2025-09-22T13:59:45.802831,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -36554,7 +36554,7 @@ function longest(strings){ } } return result;",,77.51000000280328,,Completions generated successfully. Ready for display in your IDE.,5b04de28-b8b3-4686-97c5-98ef13cbf2a6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:45.899898,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36580,7 +36580,7 @@ class Problem { } }",96.59339999780059,,Completions generated successfully. Ready for display in your IDE.,67974fcb-4f60-4100-9d63-7efc943a2c3a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T13:59:45.976963,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36605,7 +36605,7 @@ class Problem { } return ret; }",,77.15699999971548,,Completions generated successfully. Ready for display in your IDE.,3505bad2-6cd4-4ef6-af0b-0204d20f82a1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T13:59:46.036065,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -36635,7 +36635,7 @@ function count_up_to(n){ return output; } ",58.19360000168672,,Completions generated successfully. Ready for display in your IDE.,40683558-4fd4-484c-8230-f6a9edb99eb0,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:46.158432,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -36668,7 +36668,7 @@ function prod_signs(arr){ } ",122.43989999842597,,Completions generated successfully. Ready for display in your IDE.,40b6d7ba-e52b-4b36-8c19-9c17f788fd13,1,deepseek-ai/deepseek-coder-1.3b-base," var pro = 1; -",98,1.0 +",98,1.0 2025-09-22T13:59:46.215252,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -36693,7 +36693,7 @@ function string_xor(a, b){ result = arrayXOR.join(''); return result; }",56.83439999847906,,Completions generated successfully. Ready for display in your IDE.,86c647cf-d028-4e18-9247-3162c99a5f1b,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,0.5341 +",32,0.5341 2025-09-22T13:59:46.273145,[1],"//Given an array arr of integers and a positive integer k, return a sorted array // of length k with the maximum k numbers in arr. // Example 1: @@ -36713,7 +36713,7 @@ function maximum(arr, k){ var len = arr.length; if(len === 0) return null;"," return arr.sort((a, b) => a - b).slice(len - k); }",57.505699995090254,,Completions generated successfully. Ready for display in your IDE.,71ab17f2-c7d4-4699-9d1a-aa8f620b879d,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:59:46.391925,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36736,7 +36736,7 @@ class Problem { return max; } }",118.41329999879235,,Completions generated successfully. Ready for display in your IDE.,0bcdd95b-5762-4773-ad54-a1a6da2f290b,1,deepseek-ai/deepseek-coder-1.3b-base," Long max = null; -",93,0.9905 +",93,0.9905 2025-09-22T13:59:46.457865,[1],"//Return true if all numbers in the array l are below threshold t. // >>> below_threshold([1, 2, 4, 10], 100) // true @@ -36744,7 +36744,7 @@ class Problem { // false function below_threshold(l, t){ return l.every(x => x < t);",,66.02120000025025,,Completions generated successfully. Ready for display in your IDE.,843198e8-8a50-43da-900e-ba45b73322be,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:59:47.176664,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36792,7 +36792,7 @@ class Problem { copy.set(i, copy.get(j)); copy.set(j, temp); break; - ",688,0.9528 + ",688,0.9528 2025-09-22T13:59:47.342690,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36827,7 +36827,7 @@ class Problem { return ans; } }",165.53730000305222,,Completions generated successfully. Ready for display in your IDE.,8a48ee89-1ea5-4ad1-af45-f44ab55f461c,1,deepseek-ai/deepseek-coder-1.3b-base," int sz = lst.size(); -",140,1.0 +",140,1.0 2025-09-22T13:59:47.429916,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36850,7 +36850,7 @@ class Problem { } }",86.95230000012089,,Completions generated successfully. Ready for display in your IDE.,18dc5061-9b67-47c3-89b6-ce6539e7a738,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:59:47.600323,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36878,7 +36878,7 @@ class Problem { } }",170.65049999655457,,Completions generated successfully. Ready for display in your IDE.,317613bb-7357-4db3-9452-8909258d4252,1,deepseek-ai/deepseek-coder-1.3b-base," index = string.indexOf(substring); -",145,1.0 +",145,1.0 2025-09-22T13:59:47.823869,[1],"from typing import Optional def string_to_md5(text: str) -> Optional[str]: @@ -36893,7 +36893,7 @@ def string_to_md5(text: str) -> Optional[str]: if not text: return None else:",,223.33269999944605,,Completions generated successfully. Ready for display in your IDE.,01024c2e-0b19-49b8-ad7c-52653d059afb,1,deepseek-ai/deepseek-coder-1.3b-base," - return hashlib.md5(text.encode()).hexdigest()",200,1.0 + return hashlib.md5(text.encode()).hexdigest()",200,1.0 2025-09-22T13:59:48.123300,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36938,7 +36938,7 @@ class Problem { if ((Collections.min(copy)).equals(last)){ return true; } -",268,0.9212 +",268,0.9212 2025-09-22T13:59:48.253718,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36988,7 +36988,7 @@ class Problem { } }",129.9384000012651,,Completions generated successfully. Ready for display in your IDE.,311364f0-36a0-4c96-a4d7-b3c6ecc98fd2,1,deepseek-ai/deepseek-coder-1.3b-base," case 'u': -",100,1.0 +",100,1.0 2025-09-22T13:59:48.449812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37020,7 +37020,7 @@ class Problem { } }",196.00319999881322,,Completions generated successfully. Ready for display in your IDE.,51ec2233-c435-47e7-b535-360d51e071e8,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (c == ')') { -",156,1.0 +",156,1.0 2025-09-22T13:59:48.560909,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37085,7 +37085,7 @@ class Problem { } return letterGrades;",},110.66469999786932,,Completions generated successfully. Ready for display in your IDE.,8163f82c-d300-4c3d-a332-2ce2b460676d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",81,1.0 +",81,1.0 2025-09-22T13:59:48.667131,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -37108,7 +37108,7 @@ function factorize(n){ } ",106.30160000437172,,Completions generated successfully. Ready for display in your IDE.,a4359bfc-7de6-435d-893f-12d7137d4050,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",83,1.0 +",83,1.0 2025-09-22T13:59:48.793142,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -37127,7 +37127,7 @@ function choose_num(x, y){ ",125.78559999383288,,Completions generated successfully. Ready for display in your IDE.,50b474a6-2709-4591-9a97-3381478fc8de,1,deepseek-ai/deepseek-coder-1.3b-base," break; } -",101,0.9896 +",101,0.9896 2025-09-22T13:59:48.990341,[1],"//Given an array arr of integers and a positive integer k, return a sorted array // of length k with the maximum k numbers in arr. // Example 1: @@ -37148,7 +37148,7 @@ function maximum(arr, k){ if(len === 1) return arr; return arr.sort((a, b) => a - b).slice(len - k); }",197.2276000014972,,Completions generated successfully. Ready for display in your IDE.,e7f3fa0e-aef4-4cbf-bdbf-8c4a2b65f9eb,1,deepseek-ai/deepseek-coder-1.3b-base," let len = Math.min(k, arr.length); -",174,0.8343 +",174,0.8343 2025-09-22T13:59:49.086734,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -37174,7 +37174,7 @@ function maximum(arr, k){ ",95.8466000010958,,Completions generated successfully. Ready for display in your IDE.,1944de3b-7023-4a1b-8ffa-9bd7143b832b,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:59:49.183341,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -37206,7 +37206,7 @@ function correct_bracketing(brackets){ } }",95.96289999899454,,Completions generated successfully. Ready for display in your IDE.,b194f529-ec8f-4ff6-a308-aac068297cbc,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",73,0.9579 +",73,0.9579 2025-09-22T13:59:49.367582,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37243,7 +37243,7 @@ class Problem { } }",184.21349999698577,,Completions generated successfully. Ready for display in your IDE.,cb0abc55-d116-438a-a2b4-dea678f6228c,1,deepseek-ai/deepseek-coder-1.3b-base," if (open_paren > 0) { -",162,1.0 +",162,1.0 2025-09-22T13:59:50.069389,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -37287,7 +37287,7 @@ function get_closest_vowel(word){ break; } } - console.log('here', v",676,0.9233 + console.log('here', v",676,0.9233 2025-09-22T13:59:50.332054,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -37315,7 +37315,7 @@ function get_max_triples(n){ } ",262.86500000423985,,Completions generated successfully. Ready for display in your IDE.,de9614df-04f9-4dea-a7c5-a1a68e21ac26,1,deepseek-ai/deepseek-coder-1.3b-base," for(let j = i+1; j k.matches(""^[A-Z]+$"")); return allLowerCase || allUpperCase; }",,81.97980000113603,,Completions generated successfully. Ready for display in your IDE.,e1eba5b0-9ab3-48b1-86cc-556402bf3c57,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:59:50.850172,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -37381,7 +37381,7 @@ function hex_key(num){ continue; }else { // console.log('test'); -",400,0.9185 +",400,0.9185 2025-09-22T13:59:50.973705,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -37395,7 +37395,7 @@ function largest_divisor(n){ return ld; }",123.60120000084862,,Completions generated successfully. Ready for display in your IDE.,9559f20f-5b5f-4bdc-ae23-51d8e464516c,1,deepseek-ai/deepseek-coder-1.3b-base," ld = i; -",101,1.0 +",101,1.0 2025-09-22T13:59:51.018146,[1],"from typing import List def total_match(lst1: List[str], lst2: List[str]) -> List[str]: @@ -37424,7 +37424,7 @@ def total_match(lst1: List[str], lst2: List[str]) -> List[str]: return lst1 return lst2 -",44.367399998009205,,Completions generated successfully. Ready for display in your IDE.,55e454d8-a76b-490f-8efb-423941fe8f37,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",44.367399998009205,,Completions generated successfully. Ready for display in your IDE.,55e454d8-a76b-490f-8efb-423941fe8f37,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:51.113444,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37457,7 +37457,7 @@ class Problem { } }",94.74629999749595,,Completions generated successfully. Ready for display in your IDE.,5d86b795-4f2c-4366-8959-8c9bbc5ca6d9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:51.157263,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -37476,7 +37476,7 @@ def below_zero(operations: List[int]) -> bool: return False print(below_zero([1, 2, 3])) -print(below_zero([1, 2, -4, 5]))",43.77999999996973,,Completions generated successfully. Ready for display in your IDE.,8c9f668a-79df-468a-aa33-c249de663452,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +print(below_zero([1, 2, -4, 5]))",43.77999999996973,,Completions generated successfully. Ready for display in your IDE.,8c9f668a-79df-468a-aa33-c249de663452,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:51.403812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37541,7 +37541,7 @@ class Problem { } }",246.2322000064887,,Completions generated successfully. Ready for display in your IDE.,9c6f2d4e-c671-48d7-905d-c6e8c698db72,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 0.0f) { -",216,1.0 +",216,1.0 2025-09-22T13:59:51.794656,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37595,7 +37595,7 @@ class Problem { }",390.907300003164,,Completions generated successfully. Ready for display in your IDE.,8de4ab13-3730-4a3b-a654-0e28e8078414,1,deepseek-ai/deepseek-coder-1.3b-base," else if (arr.get(i) == minValue && i < minIndex) { minIndex = i; } - ",362,1.0 + ",362,1.0 2025-09-22T13:59:51.911035,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -37626,7 +37626,7 @@ function skjkasdkd(lst){ return sum; }",116.15219999657711,,Completions generated successfully. Ready for display in your IDE.,635f9ca2-f084-413a-a4b8-6407d66d1335,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",90,1.0 +",90,1.0 2025-09-22T13:59:52.063396,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37694,7 +37694,7 @@ class Problem { } }",151.5988000028301,,Completions generated successfully. Ready for display in your IDE.,44d1af72-b302-4648-9c24-cb445db5bfdc,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",105,1.0 +",105,1.0 2025-09-22T13:59:52.216552,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -37736,7 +37736,7 @@ function is_sorted(lst){ } ",153.79620000021532,,Completions generated successfully. Ready for display in your IDE.,f2011498-1418-49ab-80b1-0dbccdfeeb45,1,deepseek-ai/deepseek-coder-1.3b-base," dupe_found = true; -",130,1.0 +",130,1.0 2025-09-22T13:59:52.483031,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -37754,7 +37754,7 @@ function x_or_y(n, x, y){ return x; }",265.9473000021535,,Completions generated successfully. Ready for display in your IDE.,99d03aad-f403-4b2b-ab10-44e190c38c69,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=2; i < Math.sqrt(n); ++i ){ -",242,0.9653 +",242,0.9653 2025-09-22T13:59:52.735251,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -37780,7 +37780,7 @@ function x_or_y(n, x, y){ return ch.lower() return chr(ord(ch) + 2) return ch",,251.70410000282573,,Completions generated successfully. Ready for display in your IDE.,690c6e92-29f1-41ec-ae67-c4c8216f4ac2,1,deepseek-ai/deepseek-coder-1.3b-base," - return """".join([encode_vowel(ch) for ch in message])",226,0.9325 + return """".join([encode_vowel(ch) for ch in message])",226,0.9325 2025-09-22T13:59:52.985246,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -37835,7 +37835,7 @@ function by_length(arr){ return arr3; }",250.23979999969015,,Completions generated successfully. Ready for display in your IDE.,088b3efd-12da-4339-a82e-bf201045b1a0,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i = 0; i < arr.length; i++){ -",223,1.0 +",223,1.0 2025-09-22T13:59:53.170641,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -37879,7 +37879,7 @@ function histogram(test){ } ",185.09690000064438,,Completions generated successfully. Ready for display in your IDE.,ea8583d1-32ee-43bb-b6f5-a5b7a26a97d3,1,deepseek-ai/deepseek-coder-1.3b-base," count[lst[i]] = 1; -",160,1.0 +",160,1.0 2025-09-22T13:59:53.455212,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -37919,7 +37919,7 @@ function find_max(words){ }",283.8666000025114,,Completions generated successfully. Ready for display in your IDE.,f96fbb30-0872-4ee4-ad91-9c8c74bf6f56,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i=0; i= 50) { -",143,1.0 +",143,1.0 2025-09-22T13:59:53.801198,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -38029,7 +38029,7 @@ def histogram(test: str) -> Dict[str, int]: if letters[letter] == most_count:"," return result ",174.23430000053486,,Completions generated successfully. Ready for display in your IDE.,bde1ade6-c1e8-47e0-b303-95a25bb53848,1,deepseek-ai/deepseek-coder-1.3b-base," result[letter] = letters[letter] -",149,1.0 +",149,1.0 2025-09-22T13:59:53.901250,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -38056,7 +38056,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: for k in range(j + 1, len(l)): if l[i] + l[j] + l[k] == 0: return True",,99.46759999729693,,Completions generated successfully. Ready for display in your IDE.,db34626e-d70d-4511-89b7-1142ec7fc5fc,1,deepseek-ai/deepseek-coder-1.3b-base," - return False",72,1.0 + return False",72,1.0 2025-09-22T13:59:54.035969,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -38085,7 +38085,7 @@ function split_words(txt){ } }",135.23149999673478,,Completions generated successfully. Ready for display in your IDE.,c5a2efa6-482c-4d9f-8e21-b90665e2a54b,1,deepseek-ai/deepseek-coder-1.3b-base," let count = 0; -",102,0.9725 +",102,0.9725 2025-09-22T13:59:54.129979,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38122,7 +38122,7 @@ class Problem { } }",93.6437999989721,,Completions generated successfully. Ready for display in your IDE.,75074240-da91-4276-b731-8e017a68e59c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:54.242653,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -38194,7 +38194,7 @@ function minPath(grid, k){ return path; }",112.68980000022566,,Completions generated successfully. Ready for display in your IDE.,0b18728f-b0a7-434e-b793-36d52e5c3bfd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:54.372675,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38272,7 +38272,7 @@ class Problem { }"," } }",129.86309999541845,,Completions generated successfully. Ready for display in your IDE.,cdd36d73-b17f-43ad-bc29-f2cee52cfa4c,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",92,1.0 +",92,1.0 2025-09-22T13:59:54.468167,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38302,7 +38302,7 @@ class Problem { } }",95.31099999730941,,Completions generated successfully. Ready for display in your IDE.,c6ac14f5-3f0c-4220-83b5-3bb6cced4d89,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",72,0.9747 +",72,0.9747 2025-09-22T13:59:54.826083,[1],"//Filter an input array of strings only for ones that start with a given prefix. // >>> filter_by_prefix([], ""a"") // [] @@ -38317,7 +38317,7 @@ function filter_by_prefix(strings, prefix){ } else { return false; } -",336,0.995 +",336,0.995 2025-09-22T13:59:54.923992,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38367,7 +38367,7 @@ class Problem { } }",97.75519999675453,,Completions generated successfully. Ready for display in your IDE.,6012e7df-1a04-4d71-b66a-216efd69358d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:55.041241,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -38383,7 +38383,7 @@ class Problem { return digits(d) * (m if m % 2 == 1 else 1) ",117.64750000293134,,Completions generated successfully. Ready for display in your IDE.,fca17bca-7c41-46a9-9918-85a7a65e22f5,1,deepseek-ai/deepseek-coder-1.3b-base," return 0 -",87,0.9286 +",87,0.9286 2025-09-22T13:59:55.176836,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -38424,7 +38424,7 @@ function int_to_mini_roman(number){ } ",135.27520000207005,,Completions generated successfully. Ready for display in your IDE.,80d7d3f0-dae1-4f8b-bb52-e6f6ab91db87,1,deepseek-ai/deepseek-coder-1.3b-base," 9: ""ix"", -",111,1.0 +",111,1.0 2025-09-22T13:59:55.357235,[1],"from typing import List def will_it_fly(q: List[int], w: int) -> bool: @@ -38450,7 +38450,7 @@ def will_it_fly(q: List[int], w: int) -> bool: # 3 is less than the maximum possible weight, and it's balanced. """""" s = sum(q)",,180.14190000394592,,Completions generated successfully. Ready for display in your IDE.,e4f20935-7e15-4690-b599-cddfb3a3bda7,1,deepseek-ai/deepseek-coder-1.3b-base," - return s <= w and s % 2 == 0",156,0.9409 + return s <= w and s % 2 == 0",156,0.9409 2025-09-22T13:59:55.444156,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -38478,7 +38478,7 @@ function is_happy(s){ }"," return happy; }",86.93649999622721,,Completions generated successfully. Ready for display in your IDE.,768fb591-6799-4dc1-88cb-95ec0911b561,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:55.769356,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -38509,7 +38509,7 @@ function words_in_sentence(sentence){ }",324.52439999906346,,Completions generated successfully. Ready for display in your IDE.,d018c058-2b26-45c8-9140-4deda58ee98c,1,deepseek-ai/deepseek-coder-1.3b-base," else{ for (var i = 3; i < Math.sqrt(number); i += 2){ - ",298,0.8906 + ",298,0.8906 2025-09-22T13:59:56.249988,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -38540,7 +38540,7 @@ function will_it_fly(q, w){ //check to see whether all values are the same in reverse order for(let i = 0; i < Math.floor(q.length/2); i++){ -",456,0.9073 +",456,0.9073 2025-09-22T13:59:56.327020,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38576,7 +38576,7 @@ class Problem { } return count; }",,76.3098000024911,,Completions generated successfully. Ready for display in your IDE.,722d883c-ba2d-4406-835d-1ec13537d669,1,deepseek-ai/deepseek-coder-1.3b-base," -}",46,1.0 +}",46,1.0 2025-09-22T13:59:56.472254,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -38600,7 +38600,7 @@ class Problem { ",145.345199998701,,Completions generated successfully. Ready for display in your IDE.,6a09d0d5-b4d1-48b5-a9ad-c71aff0c7d49,1,deepseek-ai/deepseek-coder-1.3b-base," for char in brackets: -",121,1.0 +",121,1.0 2025-09-22T13:59:56.621305,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -38677,7 +38677,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",148.82650000072317,,Completions generated successfully. Ready for display in your IDE.,95d4188d-38df-4452-9d14-579739e99d5e,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",119,1.0 +",119,1.0 2025-09-22T13:59:56.747416,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38725,7 +38725,7 @@ class Problem { } }",125.61899999855086,,Completions generated successfully. Ready for display in your IDE.,ed4bd068-822c-4b74-8c3a-f15bf18a648c,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",98,1.0 +",98,1.0 2025-09-22T13:59:57.006980,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38773,7 +38773,7 @@ class Problem { } }",259.9149000016041,,Completions generated successfully. Ready for display in your IDE.,96c28828-3b2a-48bb-bb5d-b8140da27e4d,1,deepseek-ai/deepseek-coder-1.3b-base," if (l.get(i) % 3 != 0) { -",232,0.9833 +",232,0.9833 2025-09-22T13:59:57.071816,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38795,7 +38795,7 @@ class Problem { BigInteger nSquared = nBigInteger.multiply(nBigInteger); return nSquared.longValue(); }",,64.60409999999683,,Completions generated successfully. Ready for display in your IDE.,1a6ae691-0dc1-406b-9d09-7b01a535bb71,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:59:57.180635,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -38823,7 +38823,7 @@ function triples_sum_to_zero(l){ } ",108.53279999719234,,Completions generated successfully. Ready for display in your IDE.,1c927710-5776-41dd-ace8-6924b187f25b,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:59:57.771323,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -38864,7 +38864,7 @@ function find_max(words){ //console.log(""Unique: "", JSON.stringify(unique_word)); if(!max_unique || (Object.keys(unique_word).length >= Object.keys(max_unique).length)){ -",563,0.9664 +",563,0.9664 2025-09-22T13:59:57.898155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38892,7 +38892,7 @@ class Problem { return p; } }",126.71039999986533,,Completions generated successfully. Ready for display in your IDE.,2f157aee-340a-4b75-ace2-cc68e1add9f2,1,deepseek-ai/deepseek-coder-1.3b-base," Long p = 0L; -",102,0.9586 +",102,0.9586 2025-09-22T13:59:58.099530,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38928,7 +38928,7 @@ class Problem { } }",200.70159999886528,,Completions generated successfully. Ready for display in your IDE.,397961c8-0692-4fd4-ad90-f885cbb7b958,1,deepseek-ai/deepseek-coder-1.3b-base," else n1 = 3 * n1 + 1; -",173,1.0 +",173,1.0 2025-09-22T13:59:58.260837,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -38986,7 +38986,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",161.36939999705646,,Completions generated successfully. Ready for display in your IDE.,7b3f8379-e6de-438e-8ccb-c2ca132ba1b4,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 1.7: -",131,1.0 +",131,1.0 2025-09-22T13:59:58.379491,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -39021,7 +39021,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: roman_numeral += roman_numerals[key] * (number // key) number = number % key"," ",118.67849999543978,,Completions generated successfully. Ready for display in your IDE.,fdef8d23-c07f-4015-b788-466b371e6551,1,deepseek-ai/deepseek-coder-1.3b-base," - return roman_numeral",94,1.0 + return roman_numeral",94,1.0 2025-09-22T13:59:58.477863,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -39053,7 +39053,7 @@ function correct_bracketing(brackets){ } }",98.1541999935871,,Completions generated successfully. Ready for display in your IDE.,fb28d913-889c-4c63-946a-b6e37a338228,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",72,1.0 +",72,1.0 2025-09-22T13:59:58.553259,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -39079,7 +39079,7 @@ function correct_bracketing(brackets){ } ",74.8576999976649,,Completions generated successfully. Ready for display in your IDE.,e5e8253e-7292-4deb-b46d-f48e240e1a33,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,1.0 +",53,1.0 2025-09-22T13:59:58.639425,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39108,7 +39108,7 @@ class Problem { } return (sqrt * 2) % p;",},85.95390000118641,,Completions generated successfully. Ready for display in your IDE.,e0a736fb-3cef-4155-93ca-6abfa0f195b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:58.708452,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -39118,7 +39118,7 @@ function all_prefixes(string){ result.push(string.slice(0, i)); } return result;",,68.86270000541117,,Completions generated successfully. Ready for display in your IDE.,030f2160-fcc0-4ae7-985f-57d38afd0c4f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,0.9043 +}",47,0.9043 2025-09-22T13:59:58.789025,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39185,7 +39185,7 @@ class Problem { } } }",80.26969999627909,,Completions generated successfully. Ready for display in your IDE.,520a2cf6-ca7a-48fe-846a-0c32c12008f9,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:59:58.978622,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -39252,7 +39252,7 @@ function numerical_letter_grade(grades){ return grades_array; }",189.20399999478832,,Completions generated successfully. Ready for display in your IDE.,c8185637-7f32-44a3-a06c-8c0fe2220140,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""A-""); -",160,1.0 +",160,1.0 2025-09-22T13:59:59.031705,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -39281,7 +39281,7 @@ function parse_nested_parens(paren_string){ } ",52.981799999543,,Completions generated successfully. Ready for display in your IDE.,0ad69b2c-6749-4c78-8add-59d66dc927e8,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:59:59.127585,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39304,7 +39304,7 @@ class Problem { } return true;",},96.04029999900376,,Completions generated successfully. Ready for display in your IDE.,3f7025c8-0b27-4666-9b46-dfd81f1ffde7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T13:59:59.212386,[1],"//Return n-th Fibonacci number. // >>> fib(10) // 55 @@ -39319,7 +39319,7 @@ function fib(n){ return fib(n-1) + fib(n-2);","} ",84.14359999733279,,Completions generated successfully. Ready for display in your IDE.,6e3d5137-0190-4b88-97cd-49ccf0f62bac,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:59.369548,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -39374,7 +39374,7 @@ function by_length(arr){ return arr3; }",156.9078999964404,,Completions generated successfully. Ready for display in your IDE.,d112d7d9-bfec-47d6-9866-55db5158527d,1,deepseek-ai/deepseek-coder-1.3b-base," else if(num == 6){ -",130,1.0 +",130,1.0 2025-09-22T13:59:59.538872,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39416,7 +39416,7 @@ class Problem { } }",169.0870000020368,,Completions generated successfully. Ready for display in your IDE.,1209e292-f6e8-4a0f-8f6a-ce23cc017158,1,deepseek-ai/deepseek-coder-1.3b-base," StringBuilder sb = new StringBuilder(); -",143,1.0 +",143,1.0 2025-09-22T13:59:59.767085,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39484,7 +39484,7 @@ class Problem { } }",228.14970000035828,,Completions generated successfully. Ready for display in your IDE.,e7cca9c5-fe9c-42c8-8fb8-867304c09406,1,deepseek-ai/deepseek-coder-1.3b-base," seen_2_in_a_row = false; -",198,1.0 +",198,1.0 2025-09-22T13:59:59.808907,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -39506,7 +39506,7 @@ function encode(message){ result += letter.toUpperCase() === letter ? letter.toLowerCase() : letter.toUpperCase(); } return result; -}",41.81139999855077,,Completions generated successfully. Ready for display in your IDE.,ed0ee060-46f9-437f-bd0a-45559ec79371,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",41.81139999855077,,Completions generated successfully. Ready for display in your IDE.,ed0ee060-46f9-437f-bd0a-45559ec79371,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T14:00:00.091650,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39542,7 +39542,7 @@ class Problem { } }",282.7612999972189,,Completions generated successfully. Ready for display in your IDE.,b7da4953-e399-4b87-8909-9ec8e8abdaf5,1,deepseek-ai/deepseek-coder-1.3b-base," for (int j = 0; j < lst2.size(); j++) { -",255,0.9936 +",255,0.9936 2025-09-22T14:00:00.220161,[1],"from typing import Union, List def split_words(txt: str) -> Union[List[str], int]: @@ -39565,7 +39565,7 @@ def split_words(txt: str) -> Union[List[str], int]: return len([letter for letter in txt if letter in alphabet and alphabet.index(letter) % 2 == 1]) ",127.9385999951046,,Completions generated successfully. Ready for display in your IDE.,aec8d816-759f-49d1-ad84-a3013548b62a,1,deepseek-ai/deepseek-coder-1.3b-base," return txt.split() -",104,1.0 +",104,1.0 2025-09-22T14:00:00.676440,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -39599,7 +39599,7 @@ function skjkasdkd(lst){ for (let i=2; i*i <= n ; ++i ) { if (n % i == 0)return false;} -",431,0.9351 +",431,0.9351 2025-09-22T14:00:01.410197,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -39685,7 +39685,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i if __name__ == ""__main__"": import doctest - doctest",690,0.9727 + doctest",690,0.9727 2025-09-22T14:00:01.489354,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39702,7 +39702,7 @@ class Problem { public static String concatenate(ArrayList strings) { return strings.stream().collect(Collectors.joining());",},79.10419999825535,,Completions generated successfully. Ready for display in your IDE.,abd6e77f-4b61-45f4-a24a-52d4b59163e5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T14:00:01.543164,[1],"//xs represent coefficients of a polynomial. // xs[0] + xs[1] * x + xs[2] * x^2 + .... // Return derivative of this polynomial in the same form. @@ -39716,7 +39716,7 @@ function derivative(xs){ coeff.push(xs[i] * i);"," return coeff; } ",53.55749999580439,,Completions generated successfully. Ready for display in your IDE.,2e3c4590-1f55-4c2e-b4c5-811a74fd8fad,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:01.663217,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39756,7 +39756,7 @@ class Problem { Collections.sort(b);"," } }",119.9172000051476,,Completions generated successfully. Ready for display in your IDE.,c62c01f9-aa10-4c41-a9d1-6156768e5a27,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",95,1.0 +",95,1.0 2025-09-22T14:00:01.739254,[1],"//Return a greatest common divisor of two integers a and b // >>> greatest_common_divisor(3, 5) // 1 @@ -39772,7 +39772,7 @@ function greatest_common_divisor(a, b){ ",75.58070000231965,,Completions generated successfully. Ready for display in your IDE.,253108fe-07d0-41bf-b9ec-ef92a100c16c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",51,1.0 +",51,1.0 2025-09-22T14:00:01.832163,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -39796,7 +39796,7 @@ function vowels_count(s){ }","} ",92.78529999573948,,Completions generated successfully. Ready for display in your IDE.,b2f75627-5b14-47b8-b971-a6dfcd4d6d51,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",69,1.0 +",69,1.0 2025-09-22T14:00:02.030902,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -39815,7 +39815,7 @@ function triangle_area(a, b, c){ } ",198.3235000006971,,Completions generated successfully. Ready for display in your IDE.,83e8319e-42f3-4fe0-9c22-2644270525ed,1,deepseek-ai/deepseek-coder-1.3b-base," let s= (a + b + c)/2; -",176,0.9288 +",176,0.9288 2025-09-22T14:00:02.178530,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -39833,7 +39833,7 @@ function triangle_area(a, b, c){ ",147.68890000414103,,Completions generated successfully. Ready for display in your IDE.,f0735df7-3104-4ecd-91c5-109a155fd96b,1,deepseek-ai/deepseek-coder-1.3b-base," while (n != 1): -",124,0.9375 +",124,0.9375 2025-09-22T14:00:02.664807,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39865,7 +39865,7 @@ class Problem { } }",486.18010000063805,,Completions generated successfully. Ready for display in your IDE.,39c13816-b2ae-4c05-8949-68bfe586467e,1,deepseek-ai/deepseek-coder-1.3b-base," if ((guess.get(i).equals(game.get(i))) || Math.abs(guess.get(i)).equals(game.get(i))) { -",458,0.9841 +",458,0.9841 2025-09-22T14:00:02.829619,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39916,7 +39916,7 @@ class Problem { } }",164.3882999997004,,Completions generated successfully. Ready for display in your IDE.,2ebb46ef-28e5-4a97-9a1b-364707a66380,1,deepseek-ai/deepseek-coder-1.3b-base," if (count > 3) { -",131,1.0 +",131,1.0 2025-09-22T14:00:03.530781,[1],"//Complete the function that takes two integers and returns // the product of their unit digits. // Assume the input is always valid. @@ -39934,7 +39934,7 @@ function multiply(a, b){ let bStr = b.toString()","} ",700.7826999979443,,Completions generated successfully. Ready for display in your IDE.,2da7fdc6-02b1-423d-b54c-aa784545221b,1,deepseek-ai/deepseek-coder-1.3b-base," - return parseInt((aStr[aStr.length-1] * bStr[bStr.length-1])) + (parseInt(aStr[(aStr.length)-2])*parseInt(bStr[(bStr.length)-2]))+multiply(Math",668,0.9396 + return parseInt((aStr[aStr.length-1] * bStr[bStr.length-1])) + (parseInt(aStr[(aStr.length)-2])*parseInt(bStr[(bStr.length)-2]))+multiply(Math",668,0.9396 2025-09-22T14:00:03.633936,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -40001,7 +40001,7 @@ function numerical_letter_grade(grades){ return grades_array; }",102.83550000167452,,Completions generated successfully. Ready for display in your IDE.,13dd8483-b7db-4ba5-a3d0-07d0b29a4cbb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:03.761353,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -40037,7 +40037,7 @@ function numerical_letter_grade(grades){ ",127.2949999984121,,Completions generated successfully. Ready for display in your IDE.,3169d4ae-8d81-4fa7-a345-f5e82b3de3ea,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",99,1.0 +",99,1.0 2025-09-22T14:00:03.818794,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40071,7 +40071,7 @@ class Problem { return true; } }",57.42439999448834,,Completions generated successfully. Ready for display in your IDE.,e1f112d1-2d5c-4170-9683-feb2c2b30876,1,deepseek-ai/deepseek-coder-1.3b-base," -",34,1.0 +",34,1.0 2025-09-22T14:00:03.914726,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -40098,7 +40098,7 @@ class Problem { return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",95.70849999727216,,Completions generated successfully. Ready for display in your IDE.,1e9c909d-f75f-4425-88e6-894269396143,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",68,0.9167 +",68,0.9167 2025-09-22T14:00:04.214127,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -40127,7 +40127,7 @@ function solve(s){ ",298.90890000388026,,Completions generated successfully. Ready for display in your IDE.,aa06335f-426e-4894-bd39-351f2723167f,1,deepseek-ai/deepseek-coder-1.3b-base," // console.log({letter: char}); return (/\d/.test(char)) - ",269,0.9457 + ",269,0.9457 2025-09-22T14:00:04.497926,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40170,7 +40170,7 @@ class Problem { } }",283.84270000242395,,Completions generated successfully. Ready for display in your IDE.,4ad88a01-603b-4bbd-8f5f-4f7bd2ec4b2c,1,deepseek-ai/deepseek-coder-1.3b-base," long minX = ((n - 3 * d) / 2) / 2; -",260,0.9549 +",260,0.9549 2025-09-22T14:00:04.593385,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40214,7 +40214,7 @@ class Problem { } }",95.11860000202432,,Completions generated successfully. Ready for display in your IDE.,13dff80a-ee2a-42b4-b213-c25140d19ba2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:04.685771,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40246,7 +40246,7 @@ class Problem { } }",91.92819999589119,,Completions generated successfully. Ready for display in your IDE.,b1975303-85b6-4e51-aa36-f23b83f9db12,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:04.850026,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -40304,7 +40304,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",164.48269999818876,,Completions generated successfully. Ready for display in your IDE.,c3d5f30d-e201-4934-b8e6-88559bfa2896,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('C+') -",131,1.0 +",131,1.0 2025-09-22T14:00:05.077136,[1],"//sum_to_n is a function that sums numbers from 1 to n. // >>> sum_to_n(30) // 465 @@ -40323,7 +40323,7 @@ function sum_to_n(n){ } ",226.75609999714652,,Completions generated successfully. Ready for display in your IDE.,fb26efae-6a9a-4edf-af39-88c723a5696b,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=1;i<=n;++i){ -",205,1.0 +",205,1.0 2025-09-22T14:00:05.223297,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -40400,7 +40400,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",145.87639999808744,,Completions generated successfully. Ready for display in your IDE.,36d01ddb-a519-4096-bb78-0261f99a84a5,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",119,1.0 +",119,1.0 2025-09-22T14:00:05.332487,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -40438,7 +40438,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",109.0669999975944,,Completions generated successfully. Ready for display in your IDE.,3bfb80f1-2d17-413f-ad80-b52addbfdcee,1,deepseek-ai/deepseek-coder-1.3b-base," }; -",69,1.0 +",69,1.0 2025-09-22T14:00:05.401890,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -40476,7 +40476,7 @@ function find_max(words){ } return max_unique_word; }",69.32479999522911,,Completions generated successfully. Ready for display in your IDE.,da42568d-8f6b-433e-81a5-2db89d825833,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:00:05.497749,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40520,7 +40520,7 @@ class Problem { } }",95.04990000277758,,Completions generated successfully. Ready for display in your IDE.,26f9504d-f5dd-430d-9606-d8e6f62ebdcf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:05.666896,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -40549,7 +40549,7 @@ function split_words(txt){ } }",168.89660000015283,,Completions generated successfully. Ready for display in your IDE.,f8253f01-a609-42bf-a277-1b56e34ac4fa,1,deepseek-ai/deepseek-coder-1.3b-base," else if (txt.includes(',')){ -",145,1.0 +",145,1.0 2025-09-22T14:00:06.038811,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -40582,7 +40582,7 @@ function same_chars(s0, s1){ //check reverse of string 2 for(let i = 0; i < s1.length; i++){ -",340,0.9455 +",340,0.9455 2025-09-22T14:00:06.122974,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -40610,7 +40610,7 @@ function pairs_sum_to_zero(l){ } ",83.6669000054826,,Completions generated successfully. Ready for display in your IDE.,8a5171f3-53a4-43b4-bcb6-b5d003122824,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:06.233027,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -40652,7 +40652,7 @@ function compare_one(a, b){ } ",109.72860000038054,,Completions generated successfully. Ready for display in your IDE.,8719a6b1-3173-4ab4-8fb7-685b42f8bbfd,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",85,1.0 +",85,1.0 2025-09-22T14:00:06.297903,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -40684,7 +40684,7 @@ function correct_bracketing(brackets){ else { return false; }",,64.4006000002264,,Completions generated successfully. Ready for display in your IDE.,439ec4f4-bb19-452a-85e5-1aa73a56fc20,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:06.737045,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40730,7 +40730,7 @@ class Problem { }",439.50520000362303,,Completions generated successfully. Ready for display in your IDE.,4dea9f4b-9294-44a3-a5d4-56e07d1e5611,1,deepseek-ai/deepseek-coder-1.3b-base," int j = q.size() - i - 1; if (!Objects.equals(q.get(i), q.get(j))) { -",411,0.9726 +",411,0.9726 2025-09-22T14:00:06.828793,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -40774,7 +40774,7 @@ function histogram(test){ } ",91.56369999982417,,Completions generated successfully. Ready for display in your IDE.,a349120b-52de-428e-bdb4-fe3dc220d388,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:06.940128,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40794,7 +40794,7 @@ class Problem { } }",111.15060000156518,,Completions generated successfully. Ready for display in your IDE.,7c6cccc5-39d1-47e8-b9ae-edab1568dbc8,1,deepseek-ai/deepseek-coder-1.3b-base," return i; -",82,1.0 +",82,1.0 2025-09-22T14:00:07.072638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40832,7 +40832,7 @@ class Problem { } }",131.58280000061495,,Completions generated successfully. Ready for display in your IDE.,16d53a97-252b-4da9-b6f9-83559c592465,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:07.242094,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40910,7 +40910,7 @@ class Problem { } }",169.72979999991367,,Completions generated successfully. Ready for display in your IDE.,26413408-0778-458c-94d1-c32eb6b30482,1,deepseek-ai/deepseek-coder-1.3b-base," while (num >= 10) { -",142,1.0 +",142,1.0 2025-09-22T14:00:07.339703,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40955,7 +40955,7 @@ class Problem { } }",97.10500000073807,,Completions generated successfully. Ready for display in your IDE.,21df2558-0b6f-43be-b8d4-7f01f1804539,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:07.547082,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -41003,7 +41003,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",207.42329999484355,,Completions generated successfully. Ready for display in your IDE.,1fd17019-cafd-47a4-b208-d434171d066f,1,deepseek-ai/deepseek-coder-1.3b-base," min_val = grid[i][j] -",171,1.0 +",171,1.0 2025-09-22T14:00:07.602348,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -41023,7 +41023,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: return -1 ",54.86689999816008,,Completions generated successfully. Ready for display in your IDE.,ac90d796-c5c5-4a5b-aa4d-495945d47542,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:07.842897,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -41053,7 +41053,7 @@ function sort_third(l){ return x; }",240.79920000076527,,Completions generated successfully. Ready for display in your IDE.,5e8efd96-96cd-4352-89d7-3f2369068c0b,1,deepseek-ai/deepseek-coder-1.3b-base," let y=new Array(Math.floor((x.length)/3)); -",217,1.0 +",217,1.0 2025-09-22T14:00:07.927182,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -41085,7 +41085,7 @@ function correct_bracketing(brackets){ else { return false;",},84.03840000391938,,Completions generated successfully. Ready for display in your IDE.,2a85a275-f122-413c-aadf-3527163fcd75,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:08.379638,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -41103,7 +41103,7 @@ function choose_num(x, y){ } ",451.9014999968931,,Completions generated successfully. Ready for display in your IDE.,505a97de-6b58-4b3c-864d-b1ff4a335ca3,1,deepseek-ai/deepseek-coder-1.3b-base," if (i % 2 == 0 && Math.floor(Math.log10(i)) == Math.ceil(Math.log10(i))) { -",411,0.9384 +",411,0.9384 2025-09-22T14:00:08.472158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41142,7 +41142,7 @@ class Problem { } }",92.51889999723062,,Completions generated successfully. Ready for display in your IDE.,4134a6dc-2fcb-4bb9-bfa9-0c3c389b1f4e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:08.839805,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41178,7 +41178,7 @@ class Problem { return """"; } StringBuilder sb = new StringBuilder(); -",342,0.9473 +",342,0.9473 2025-09-22T14:00:08.968743,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -41226,7 +41226,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",128.1656999999541,,Completions generated successfully. Ready for display in your IDE.,52e61270-1096-4b77-b39c-8aee6a35ca82,1,deepseek-ai/deepseek-coder-1.3b-base," while True: -",98,1.0 +",98,1.0 2025-09-22T14:00:09.063447,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -41253,7 +41253,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: if s[i] == s[i + 1] or s[i] == s[i + 2] or s[i + 1] == s[i + 2]: return False"," ",94.7870999953011,,Completions generated successfully. Ready for display in your IDE.,b4aed541-dee8-44f6-9b04-0d8985081fb8,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",67,1.0 + return True",67,1.0 2025-09-22T14:00:09.178203,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -41325,7 +41325,7 @@ function minPath(grid, k){ return path; }",114.7075999979279,,Completions generated successfully. Ready for display in your IDE.,fdf9ce3d-9eb1-40c2-beb2-3cfa8b9a1cbc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T14:00:09.268418,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -41353,7 +41353,7 @@ function fib4(n){ } ",89.83269999589538,,Completions generated successfully. Ready for display in your IDE.,eb5d118b-61dd-4640-bc42-de6ec8e074fa,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:00:09.463944,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -41395,7 +41395,7 @@ function compare_one(a, b){ } ",195.19620000210125,,Completions generated successfully. Ready for display in your IDE.,b770a012-b94e-49e3-b63f-5a86e92d7215,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (a[0] < b){ -",170,1.0 +",170,1.0 2025-09-22T14:00:09.705211,[1],"from typing import List def do_algebra(operator: List[str], operand: List[int]) -> int: @@ -41430,7 +41430,7 @@ def do_algebra(operator: List[str], operand: List[int]) -> int: return eval("""".join(expr))",241.15200000233017,,Completions generated successfully. Ready for display in your IDE.,be789a8d-63ba-464b-a007-4ce7dd656eb3,1,deepseek-ai/deepseek-coder-1.3b-base," if op == ""+"": expr.append(""+"") -",214,0.9931 +",214,0.9931 2025-09-22T14:00:09.791196,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41456,7 +41456,7 @@ class Problem { } return levels;",},86.04529999865917,,Completions generated successfully. Ready for display in your IDE.,c7a5bc2f-ca22-4514-b627-fb0f139c5ced,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:09.899777,[1],"def car_race_collision(n: int) -> int: """""" Imagine a road that's a perfectly straight infinitely long line. @@ -41477,7 +41477,7 @@ class Problem { for j in range(l): if left_cars[i] == right_cars[j]:", return count,108.29830000147922,,Completions generated successfully. Ready for display in your IDE.,384918b1-2179-4f85-ab9b-acc9ca8b2bec,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:09.989336,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41508,7 +41508,7 @@ class Problem { } }",89.46520000608871,,Completions generated successfully. Ready for display in your IDE.,d98ad3a3-85fc-4c21-9522-e1ccad046b2a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:10.033153,[1],"def fibfib(n: int) -> int: """"""The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fibfib(0) == 0 @@ -41528,7 +41528,7 @@ class Problem { n -= 1 return a -",43.17379999702098,,Completions generated successfully. Ready for display in your IDE.,12316f0d-c5e5-470e-b32b-66272edc85d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.17379999702098,,Completions generated successfully. Ready for display in your IDE.,12316f0d-c5e5-470e-b32b-66272edc85d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:00:10.096162,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41569,7 +41569,7 @@ class Problem { return sb.toString(); } }",62.8045999983442,,Completions generated successfully. Ready for display in your IDE.,a0131e6a-4e62-4cdc-9d3c-7ac2746dcb32,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:00:10.235954,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -41597,7 +41597,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i if largest_negative is None or num > largest_negative:"," return (largest_negative, smallest_positive) ",140.07939999282826,,Completions generated successfully. Ready for display in your IDE.,fa8453ad-e19b-4a7e-9316-be3d165c9b7d,1,deepseek-ai/deepseek-coder-1.3b-base," largest_negative = num -",104,1.0 +",104,1.0 2025-09-22T14:00:10.317381,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41617,7 +41617,7 @@ class Problem { } }",81.14039999782108,,Completions generated successfully. Ready for display in your IDE.,096ab786-450d-4947-853c-f1e3d7780109,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T14:00:10.423719,[1],"//You're given an array of deposit and withdrawal operations on a bank account that starts with // zero balance. Your task is to detect if at any point the balance of account fallls below zero, and // at that point function should return true. Otherwise it should return false. @@ -41635,7 +41635,7 @@ function below_zero(operations){ } ",106.11229999631178,,Completions generated successfully. Ready for display in your IDE.,b8afb90b-6375-4d4d-8b4e-3becc79872b0,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T14:00:10.598744,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -41707,7 +41707,7 @@ function minPath(grid, k){ return path; }",175.0746000034269,,Completions generated successfully. Ready for display in your IDE.,e9bd0666-dac5-4afa-9949-d99e4a080f32,1,deepseek-ai/deepseek-coder-1.3b-base," path.push(startCell); -",146,1.0 +",146,1.0 2025-09-22T14:00:10.750058,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -41775,7 +41775,7 @@ function do_algebra(operator, operand){ } ",150.9479000014835,,Completions generated successfully. Ready for display in your IDE.,b58ddf50-5109-4a35-9b89-75c26e49d0f1,1,deepseek-ai/deepseek-coder-1.3b-base," do_again = true; -",123,1.0 +",123,1.0 2025-09-22T14:00:10.979306,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -41819,7 +41819,7 @@ function sorted_list_sum(lst){ }); } var a = [""aa"", ""a"", ""aaa""];",,228.8929000060307,,Completions generated successfully. Ready for display in your IDE.,d17cf00c-21a6-4b9d-85c0-36ca72c845c0,1,deepseek-ai/deepseek-coder-1.3b-base," -console.log(""Result: "",sorted_list_sum(a));",203,0.9218 +console.log(""Result: "",sorted_list_sum(a));",203,0.9218 2025-09-22T14:00:11.065655,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41836,7 +41836,7 @@ class Problem { public static long strlen(String string) { return string.length();",},86.27139999589417,,Completions generated successfully. Ready for display in your IDE.,4c9083e2-c0d4-4da3-bd95-5fa294c747cd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T14:00:11.148950,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -41873,7 +41873,7 @@ function is_prime(n){ return true; } ",83.07590000185883,,Completions generated successfully. Ready for display in your IDE.,cad2257e-2f8f-4536-a84e-2196caa7d952,1,deepseek-ai/deepseek-coder-1.3b-base," - }",60,1.0 + }",60,1.0 2025-09-22T14:00:11.392927,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -41913,7 +41913,7 @@ function match_parens(lst){ return 'No'; }",243.87739999656333,,Completions generated successfully. Ready for display in your IDE.,4732aacf-e0d1-4a69-b214-2faa8beb0724,1,deepseek-ai/deepseek-coder-1.3b-base," if (!stack || !stack.slice(-1)[0] == open){ -",219,0.9911 +",219,0.9911 2025-09-22T14:00:11.758385,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41944,7 +41944,7 @@ class Problem { } }",365.48929999844404,,Completions generated successfully. Ready for display in your IDE.,691e5e34-216a-4a22-8bb8-9291de0fd7b9,1,deepseek-ai/deepseek-coder-1.3b-base," if ((result.charAt(i) == '_') && (result.charAt(i+1) == ' ')) { -",341,0.9723 +",341,0.9723 2025-09-22T14:00:11.831946,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41991,7 +41991,7 @@ class Problem { return l_s; } }",72.47119999374263,,Completions generated successfully. Ready for display in your IDE.,62045d30-a3ca-424e-b466-f1e2332e0581,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,1.0 +",46,1.0 2025-09-22T14:00:12.525842,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42024,7 +42024,7 @@ class Problem { memo[0] = 0; memo[1] = 0; memo[2] = 2; - memo[",670,0.9273 + memo[",670,0.9273 2025-09-22T14:00:12.696771,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -42044,7 +42044,7 @@ function unique_digits(x){ ",170.80469999928027,,Completions generated successfully. Ready for display in your IDE.,ea038172-cc39-48c4-a04a-d9abb5988526,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(Number(num)) -",134,0.9844 +",134,0.9844 2025-09-22T14:00:12.751946,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -42064,7 +42064,7 @@ function specialFilter(nums){ } return count; } -",54.95290000544628,,Completions generated successfully. Ready for display in your IDE.,cf34aa1c-396f-4a64-b0b6-7fca8aeaafb8,1,deepseek-ai/deepseek-coder-1.3b-base, ,31,0.8564 +",54.95290000544628,,Completions generated successfully. Ready for display in your IDE.,cf34aa1c-396f-4a64-b0b6-7fca8aeaafb8,1,deepseek-ai/deepseek-coder-1.3b-base, ,31,0.8564 2025-09-22T14:00:12.907961,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42099,7 +42099,7 @@ class Problem { } }",155.51330000016605,,Completions generated successfully. Ready for display in your IDE.,2f734107-8e86-47f4-8112-6bf587df742a,1,deepseek-ai/deepseek-coder-1.3b-base," long ans = 0L; -",129,1.0 +",129,1.0 2025-09-22T14:00:13.067127,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42120,7 +42120,7 @@ class Problem { } }",159.1855000006035,,Completions generated successfully. Ready for display in your IDE.,0f1b754b-49dc-4966-8fb6-91a883178a02,1,deepseek-ai/deepseek-coder-1.3b-base," result += Long.toString(i); -",134,1.0 +",134,1.0 2025-09-22T14:00:13.109395,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -42135,7 +42135,7 @@ function how_many_times(string, substring){ i = string.indexOf(substring, i) + 1; } return count; -}",41.99769999831915,,Completions generated successfully. Ready for display in your IDE.,2cfd41ac-1083-444c-949b-a7d2b0fe8017,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5 +}",41.99769999831915,,Completions generated successfully. Ready for display in your IDE.,2cfd41ac-1083-444c-949b-a7d2b0fe8017,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5 2025-09-22T14:00:13.192606,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42152,7 +42152,7 @@ class Problem { public static long add(long x, long y) { return x + y;",},83.22790000238456,,Completions generated successfully. Ready for display in your IDE.,75f30728-a6be-4192-8ba9-291901a29512,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:13.285737,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -42201,7 +42201,7 @@ function check_dict_case(dict){ return true; }",92.90850000252249,,Completions generated successfully. Ready for display in your IDE.,507b9a85-b25e-4436-a0c7-27244834871b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T14:00:13.412155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42269,7 +42269,7 @@ class Problem { } }",125.97100000130013,,Completions generated successfully. Ready for display in your IDE.,be465573-4a31-4a3d-a2fc-e28bf9d72fe3,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",94,1.0 +",94,1.0 2025-09-22T14:00:13.543104,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -42337,7 +42337,7 @@ function do_algebra(operator, operand){ } ",130.39450000360375,,Completions generated successfully. Ready for display in your IDE.,3bb01e86-01f4-4130-a3cf-d83c31825534,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0; -",103,1.0 +",103,1.0 2025-09-22T14:00:13.630201,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -42372,7 +42372,7 @@ function search(lst){ } ",87.28919999703066,,Completions generated successfully. Ready for display in your IDE.,5c93da8c-2164-4d0d-8469-9690fae782d7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:13.723542,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42407,7 +42407,7 @@ class Problem { } }",92.99450000253273,,Completions generated successfully. Ready for display in your IDE.,40afe20f-f951-4848-bee0-303ab6cf0d7f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:13.801593,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42430,7 +42430,7 @@ class Problem { } return ans; }",,77.6112000021385,,Completions generated successfully. Ready for display in your IDE.,829d1eb5-69be-4a07-aa13-a53736dbadaf,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:13.861962,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42468,7 +42468,7 @@ class Problem { return res; } }",60.07429999590386,,Completions generated successfully. Ready for display in your IDE.,73c7c3c5-7fea-4069-8acd-825027b93807,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T14:00:14.020929,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42546,7 +42546,7 @@ class Problem { } }",159.32080000493443,,Completions generated successfully. Ready for display in your IDE.,5f7724b1-2e85-494e-9b3e-36c18270fcf4,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 500; -",133,1.0 +",133,1.0 2025-09-22T14:00:14.311494,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -42576,7 +42576,7 @@ function closest_integer(value){ } ",290.10489999927813,,Completions generated successfully. Ready for display in your IDE.,852e78cc-c96c-4b25-abf0-1e1948d8b434,1,deepseek-ai/deepseek-coder-1.3b-base," if(Math.abs(-value + 0.5) < Math.abs((+value))){ -",266,0.925 +",266,0.925 2025-09-22T14:00:14.404371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42613,7 +42613,7 @@ class Problem { } }",92.8801999980351,,Completions generated successfully. Ready for display in your IDE.,b51fca1a-2b13-4924-ad88-b751c3d45edc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:14.467402,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42652,7 +42652,7 @@ class Problem { return words.get(0); } }",62.201300002925564,,Completions generated successfully. Ready for display in your IDE.,6cb44fc5-59a7-4b28-8e5f-32fc026e9fd5,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T14:00:14.595913,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42682,7 +42682,7 @@ class Problem { } }",129.12320000032196,,Completions generated successfully. Ready for display in your IDE.,df21c070-c54b-4439-b390-bebf8b9d08d9,1,deepseek-ai/deepseek-coder-1.3b-base," n /= d; -",105,1.0 +",105,1.0 2025-09-22T14:00:14.916038,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -42721,7 +42721,7 @@ function max_fill(grid, capacity){ //loop through rows for (var i = 0; i < r; i++ ){ -",295,0.9972 +",295,0.9972 2025-09-22T14:00:15.191945,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42757,7 +42757,7 @@ class Problem { } }",275.5455000005895,,Completions generated successfully. Ready for display in your IDE.,fbde3beb-d17c-40a2-9167-dd4d24caac61,1,deepseek-ai/deepseek-coder-1.3b-base," List ans = Arrays.asList(odds_arr); -",245,1.0 +",245,1.0 2025-09-22T14:00:15.316371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42796,7 +42796,7 @@ class Problem { return result; } }",124.30180000228574,,Completions generated successfully. Ready for display in your IDE.,7364a3dd-c6d5-414b-bf7b-87c9a9b3bfdc,1,deepseek-ai/deepseek-coder-1.3b-base," int result = 0; -",100,1.0 +",100,1.0 2025-09-22T14:00:15.589100,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -42825,7 +42825,7 @@ class Problem { elif ch == ' ': return '_' raise ValueError(""Invalid character"") -",248,0.9611 +",248,0.9611 2025-09-22T14:00:15.661508,[1],"//Write a function that takes a string and returns true if the string // length is a prime number or false otherwise // Examples @@ -42845,7 +42845,7 @@ function prime_length(string){ } else { return false; }",,72.19650000479305,,Completions generated successfully. Ready for display in your IDE.,4ad07b3d-1a84-47a7-9a46-fa71b6610d41,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:00:15.748508,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42879,7 +42879,7 @@ class Problem { } }",87.1329000001424,,Completions generated successfully. Ready for display in your IDE.,4559fc9d-494f-49b0-8452-ecc723fa6b39,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:16.029934,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42901,7 +42901,7 @@ class Problem { } }",281.336599997303,,Completions generated successfully. Ready for display in your IDE.,30829026-b93a-439d-8031-6e9e5e8111dc,1,deepseek-ai/deepseek-coder-1.3b-base," BigInteger nSquared = (nBigInteger).multiply((nBigInteger)); -",258,0.946 +",258,0.946 2025-09-22T14:00:16.126545,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -42927,7 +42927,7 @@ class Problem { return '' ",96.13520000129938,,Completions generated successfully. Ready for display in your IDE.,d0f64d1e-a025-4ffa-92d4-02c6030a0965,1,deepseek-ai/deepseek-coder-1.3b-base," return '' -",72,1.0 +",72,1.0 2025-09-22T14:00:16.356997,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42971,7 +42971,7 @@ class Problem { } }",229.75200000655605,,Completions generated successfully. Ready for display in your IDE.,c434b741-7e6e-4f60-9ac1-af02b1e50eee,1,deepseek-ai/deepseek-coder-1.3b-base," long firstDigit = h.getFirstDigit(x); -",205,1.0 +",205,1.0 2025-09-22T14:00:16.465034,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -43006,7 +43006,7 @@ def match_parens(lst: List[str]) -> str: close_parens += 1 if open_parens == 0 and close_parens == 0: return 'Yes'",,108.11709999688901,,Completions generated successfully. Ready for display in your IDE.,a59e286a-edf2-4c44-b60c-5e3eb5529c46,1,deepseek-ai/deepseek-coder-1.3b-base," - return 'No'",84,0.965 + return 'No'",84,0.965 2025-09-22T14:00:16.644342,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -43038,7 +43038,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",178.9883000019472,,Completions generated successfully. Ready for display in your IDE.,356dcef8-bac6-4263-8216-3d673d950fa3,1,deepseek-ai/deepseek-coder-1.3b-base," if letters[letter] == most_count: -",155,1.0 +",155,1.0 2025-09-22T14:00:16.825095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43079,7 +43079,7 @@ class Problem { }",180.2112000004854,,Completions generated successfully. Ready for display in your IDE.,e6c2f335-780c-4f5e-a3f1-b086881bb563,1,deepseek-ai/deepseek-coder-1.3b-base," else { return compareResult; } -",151,1.0 +",151,1.0 2025-09-22T14:00:16.905908,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43111,7 +43111,7 @@ class Problem { } return false; }",,81.18430000467924,,Completions generated successfully. Ready for display in your IDE.,0fa8cd08-ea60-4b74-a065-29a0b35ab564,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T14:00:16.972623,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -43127,7 +43127,7 @@ function has_close_elements(numbers, threshold){ } } return false;",,66.36520000029122,,Completions generated successfully. Ready for display in your IDE.,537d1c0b-4aff-4f27-abd7-db5a327d647d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T14:00:17.081879,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43152,7 +43152,7 @@ class Problem { result = set.size();"," } }",108.97020000265911,,Completions generated successfully. Ready for display in your IDE.,02f08fb6-e974-41b3-9c34-033bc7b84b27,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",83,1.0 +",83,1.0 2025-09-22T14:00:17.179195,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43203,7 +43203,7 @@ class Problem { } }",96.31419999641366,,Completions generated successfully. Ready for display in your IDE.,1eb7eb54-781c-4b70-87e5-27db8b1dfba4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:17.503262,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43230,7 +43230,7 @@ class Problem { return -1; } }",324.5335999963572,,Completions generated successfully. Ready for display in your IDE.,8d38e1e5-f582-4a97-9c8d-c4b186291beb,1,deepseek-ai/deepseek-coder-1.3b-base," if ((a < b + c) && (b < a + c) && (c < a + b)) { -",299,0.9573 +",299,0.9573 2025-09-22T14:00:17.757252,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -43264,7 +43264,7 @@ function count_nums(arr){ return count; }",253.74149999697693,,Completions generated successfully. Ready for display in your IDE.,6f648d6e-ecec-4647-8cf5-408df0ed5eb3,1,deepseek-ai/deepseek-coder-1.3b-base," for(var k = 0; k < str.length; k++) { -",230,1.0 +",230,1.0 2025-09-22T14:00:17.893210,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -43300,7 +43300,7 @@ function count_nums(arr){ ",135.65949999610893,,Completions generated successfully. Ready for display in your IDE.,0071fd05-2108-422a-8308-c96288e3a84a,1,deepseek-ai/deepseek-coder-1.3b-base," e_count += 1 -",110,1.0 +",110,1.0 2025-09-22T14:00:18.063735,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -43331,7 +43331,7 @@ function count_up_to(n){ } ",170.19319999963045,,Completions generated successfully. Ready for display in your IDE.,db8e5bbe-8600-4179-8394-530c387504e3,1,deepseek-ai/deepseek-coder-1.3b-base," if ((i % j === 0)){ -",138,0.9814 +",138,0.9814 2025-09-22T14:00:18.504801,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43375,7 +43375,7 @@ class Problem { } }",441.08490000508027,,Completions generated successfully. Ready for display in your IDE.,b295fbb9-5a8d-4b1b-8ad1-e0f29a9c2f9c,1,deepseek-ai/deepseek-coder-1.3b-base," list.set(list.size() - 1, list.get(list.size() - 1) / operand.get(i + 1)); -",414,1.0 +",414,1.0 2025-09-22T14:00:18.734937,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43414,7 +43414,7 @@ class Problem { } }",229.3569000030402,,Completions generated successfully. Ready for display in your IDE.,41797d63-62a8-4c01-b789-617766960ae3,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList evenIndexed = new ArrayList(); -",203,1.0 +",203,1.0 2025-09-22T14:00:18.871599,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -43442,7 +43442,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",137.0463000057498,,Completions generated successfully. Ready for display in your IDE.,5a09fdf9-be53-45e0-b667-c5a5d642d05b,1,deepseek-ai/deepseek-coder-1.3b-base," for extension in extensions: -",111,1.0 +",111,1.0 2025-09-22T14:00:19.031992,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -43500,7 +43500,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.5690000031027,,Completions generated successfully. Ready for display in your IDE.,46490d0a-1af0-473d-a4d8-60aa5ef33277,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('B+') -",131,1.0 +",131,1.0 2025-09-22T14:00:19.147838,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -43531,7 +43531,7 @@ function same_chars(s0, s1){ ",116.13710000528954,,Completions generated successfully. Ready for display in your IDE.,8560ae4b-a051-44c8-96c2-ee15923bc1e3,1,deepseek-ai/deepseek-coder-1.3b-base," res = false; -",93,1.0 +",93,1.0 2025-09-22T14:00:19.274430,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43573,7 +43573,7 @@ class Problem { } }",126.72390000079758,,Completions generated successfully. Ready for display in your IDE.,f841f7f4-2bb4-4b9c-99db-4f64d742834a,1,deepseek-ai/deepseek-coder-1.3b-base," strongest = s; -",100,1.0 +",100,1.0 2025-09-22T14:00:19.352879,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -43623,7 +43623,7 @@ function prime_fib(n){ fibNum++; } return fib(fibNum - 1);",,77.84540000284323,,Completions generated successfully. Ready for display in your IDE.,c13d4be5-151a-49cd-b211-d74f8c61209d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T14:00:19.421906,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -43648,7 +43648,7 @@ function string_xor(a, b){ result = arrayXOR.join(''); return result; }",68.41820000408916,,Completions generated successfully. Ready for display in your IDE.,bd87dc44-9c8e-4ba5-bb20-8d88bcb20191,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,1.0 + ",42,1.0 2025-09-22T14:00:19.472649,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -43680,7 +43680,7 @@ function digits(n){ } ",50.77299999538809,,Completions generated successfully. Ready for display in your IDE.,075675dd-9b36-4be0-bb70-2267aa6fffb3,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:19.778067,[1],"def simplify(x: str, n: str) -> bool: """"""Your task is to implement a function that will simplify the expression x * n. The function returns True if x * n evaluates to a whole number and False @@ -43700,7 +43700,7 @@ function digits(n){ ",305.5038999955286,,Completions generated successfully. Ready for display in your IDE.,6727bc54-7a91-4b93-9814-2b3244dd2bfb,1,deepseek-ai/deepseek-coder-1.3b-base," n_num, n_denom = [int(i) for i in n.split('/')] -",283,1.0 +",283,1.0 2025-09-22T14:00:20.475263,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43726,7 +43726,7 @@ class Problem { int count = 0; for (float i : lst) { if ((i % 2 != 0) && (i > 0 || i < 0)) { - float square = Float.valueOf("""" + Math",671,0.9644 + float square = Float.valueOf("""" + Math",671,0.9644 2025-09-22T14:00:20.737931,[1],"//You will be given the name of a class (a string) and an array of extensions. // The extensions are to be used to load additional classes to the class. The // strength of the extension is as follows: Let CAP be the number of the uppercase @@ -43755,7 +43755,7 @@ function Strongest_Extension(class_name, extensions){ } ",262.7569000032963,,Completions generated successfully. Ready for display in your IDE.,e4db67c2-3c1d-4e13-9668-c32ab2abdb15,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i>> is_equal_to_sum_even(4) @@ -43824,7 +43824,7 @@ function is_equal_to_sum_even(n){ return false; } }",84.23769999353681,,Completions generated successfully. Ready for display in your IDE.,0f24734c-8dfe-40f0-a337-39089637c733,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T14:00:21.103283,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -43854,7 +43854,7 @@ function get_row(lst, x){ } ",176.20739999983925,,Completions generated successfully. Ready for display in your IDE.,e7bd4c05-8757-4a23-92a4-51eb793cc8c8,1,deepseek-ai/deepseek-coder-1.3b-base," coordinates.push([i, j]); -",141,1.0 +",141,1.0 2025-09-22T14:00:21.207674,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43886,7 +43886,7 @@ class Problem { } }",104.32150000269758,,Completions generated successfully. Ready for display in your IDE.,8d9e7e36-7e9d-41d5-9165-108d33c897f7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:21.321384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43919,7 +43919,7 @@ class Problem { }"," } }",113.15930000273511,,Completions generated successfully. Ready for display in your IDE.,2450e5c9-9bf1-4efe-abef-691533a3f761,1,deepseek-ai/deepseek-coder-1.3b-base," return ret; -",89,1.0 +",89,1.0 2025-09-22T14:00:21.409384,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -43940,7 +43940,7 @@ def odd_count(lst: List[str]) -> List[str]: if int(lst[i][j]) % 2 != 0: count += 1 output.append(f""the number of odd elements {count}n the str{count}ng {count} of the {count}nput."")",,87.82709999650251,,Completions generated successfully. Ready for display in your IDE.,15eba87c-3a23-4973-af92-1bf2fe2a119f,1,deepseek-ai/deepseek-coder-1.3b-base," - return output",61,1.0 + return output",61,1.0 2025-09-22T14:00:21.758387,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43985,7 +43985,7 @@ class Problem { } }",348.6722999950871,,Completions generated successfully. Ready for display in your IDE.,0e943648-bbd4-4bb3-a3e1-5e9f657ae3cc,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < Math.floor(q.size() / 2); i++) { -",322,1.0 +",322,1.0 2025-09-22T14:00:21.896387,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -44017,7 +44017,7 @@ function correct_bracketing(brackets){ } }",137.77309999568388,,Completions generated successfully. Ready for display in your IDE.,643b602d-9b09-4193-b10c-2f07d148111e,1,deepseek-ai/deepseek-coder-1.3b-base," opens.push('('); -",115,0.9736 +",115,0.9736 2025-09-22T14:00:22.055691,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -44044,7 +44044,7 @@ function next_smallest(lst){ } return second_smallest; }",158.65409999969415,,Completions generated successfully. Ready for display in your IDE.,6af2a98f-baaa-4e08-9b5c-8689462a167a,1,deepseek-ai/deepseek-coder-1.3b-base," if(!Array.isArray(lst)){ -",134,1.0 +",134,1.0 2025-09-22T14:00:22.379904,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -44121,7 +44121,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return a elif float(a_tmp) < b: return b",324.1459999990184,,Completions generated successfully. Ready for display in your IDE.,4e9b2c57-e984-4746-86f3-f958633cb065,1,deepseek-ai/deepseek-coder-1.3b-base," if isinstance(a, (float, int)) and isinstance(b, (float, int)): -",296,0.9914 +",296,0.9914 2025-09-22T14:00:22.462492,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44148,7 +44148,7 @@ class Problem { return Problem.fib((n - 1)) + Problem.fib((n - 2)); } }",,82.44500000000698,,Completions generated successfully. Ready for display in your IDE.,ebcd9951-7ed8-4a78-83ec-ea0d0b5710cc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:22.662500,[1],"from typing import List def has_close_elements(numbers: List[float], threshold: float) -> bool: @@ -44165,7 +44165,7 @@ def has_close_elements(numbers: List[float], threshold: float) -> bool: return False ",199.70359999570064,,Completions generated successfully. Ready for display in your IDE.,04a2478d-dc03-428f-98cb-319a15216b56,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(numbers)): -",154,0.996 +",154,0.996 2025-09-22T14:00:22.766925,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -44232,7 +44232,7 @@ function numerical_letter_grade(grades){ return grades_array; }",104.4700999991619,,Completions generated successfully. Ready for display in your IDE.,2af01c75-8c57-4fd4-8a3a-cf36bb75a066,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T14:00:22.935620,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -44258,7 +44258,7 @@ function numerical_letter_grade(grades){ ",168.28869999881135,,Completions generated successfully. Ready for display in your IDE.,58e47908-8e85-4105-af8c-d406f6a7fa0c,1,deepseek-ai/deepseek-coder-1.3b-base," prime_factors.append(n) -",145,1.0 +",145,1.0 2025-09-22T14:00:23.112475,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -44296,7 +44296,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",176.7887999958475,,Completions generated successfully. Ready for display in your IDE.,e54c04be-ac62-4c90-a601-2fd1b8b6c9fc,1,deepseek-ai/deepseek-coder-1.3b-base," if(isPalindrome(i)){ -",151,1.0 +",151,1.0 2025-09-22T14:00:23.210171,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44341,7 +44341,7 @@ class Problem { } }",97.2676000019419,,Completions generated successfully. Ready for display in your IDE.,8fbaeb9f-c9f1-44bd-819e-ed3499a659c3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:23.283835,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44358,7 +44358,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { return (new ArrayList(l.stream().filter(x -> x > 0).collect(Collectors.toList()))); }",,73.12609999644337,,Completions generated successfully. Ready for display in your IDE.,79de464d-fbab-4a63-8db6-1789f6d4d43c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,1.0 +}",50,1.0 2025-09-22T14:00:23.327875,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -44377,7 +44377,7 @@ def below_zero(operations: List[int]) -> bool: return True return False -print(below_zero([1, 2, 3]))",,44.296799998846836,,Completions generated successfully. Ready for display in your IDE.,be04dd2e-8675-4b2f-8c8a-206f26c72559,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 +print(below_zero([1, 2, 3]))",,44.296799998846836,,Completions generated successfully. Ready for display in your IDE.,be04dd2e-8675-4b2f-8c8a-206f26c72559,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 2025-09-22T14:00:23.413873,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44400,7 +44400,7 @@ class Problem { } }",85.95689999492606,,Completions generated successfully. Ready for display in your IDE.,07b1aa30-3900-4ffe-bb4e-dde1c9aa8993,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:23.519877,[1],"from typing import List def double_the_difference(lst: List[float]) -> int: @@ -44426,7 +44426,7 @@ def double_the_difference(lst: List[float]) -> int: ",105.24930000246968,,Completions generated successfully. Ready for display in your IDE.,77a70e49-2ed5-4db1-98f5-6bdb0e9da198,1,deepseek-ai/deepseek-coder-1.3b-base," sum = 0 -",74,1.0 +",74,1.0 2025-09-22T14:00:23.757874,[1],"//Return n-th Fibonacci number. // >>> fib(10) // 55 @@ -44441,7 +44441,7 @@ function fib(n){ } ",237.59940000309143,,Completions generated successfully. Ready for display in your IDE.,7d3dc8b6-48b1-402b-a922-2276b04c4dd8,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(n - 1) + fib(n - 2); -",217,0.9898 +",217,0.9898 2025-09-22T14:00:23.907844,[1],"from typing import Tuple def bf(planet1: str, planet2: str) -> Tuple[str, ...]: @@ -44472,7 +44472,7 @@ def bf(planet1: str, planet2: str) -> Tuple[str, ...]: planets = planets[start+1:end] else: planets = planets[end+1:start]",,149.00409999972908,,Completions generated successfully. Ready for display in your IDE.,1497d21a-90e6-4570-96c9-d93dd29760c8,1,deepseek-ai/deepseek-coder-1.3b-base," - return tuple(planets)",121,0.8948 + return tuple(planets)",121,0.8948 2025-09-22T14:00:24.019604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44550,7 +44550,7 @@ class Problem { } }",112.05980000522686,,Completions generated successfully. Ready for display in your IDE.,a14a1ba5-3646-44ad-aa94-ae8f214bfa2f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",75,1.0 +",75,1.0 2025-09-22T14:00:24.678564,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44603,7 +44603,7 @@ class Problem { } else { if (day < 1 || day > 29) return valid_date; } -",635,0.9866 +",635,0.9866 2025-09-22T14:00:24.944065,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -44644,7 +44644,7 @@ function int_to_mini_roman(number){ }","} ",264.9245000066003,,Completions generated successfully. Ready for display in your IDE.,70f2580e-24ad-4f16-ac94-48b1a02fd56a,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""The mini version of the given number is "",output,"".""); -",224,0.9035 +",224,0.9035 2025-09-22T14:00:25.026883,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44672,7 +44672,7 @@ class Problem { } return p; }",,83.006300003035,,Completions generated successfully. Ready for display in your IDE.,af0e25e9-8f01-4f18-b619-8e4c264fad35,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:25.165661,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -44705,7 +44705,7 @@ function prod_signs(arr){ } ",138.28230000217445,,Completions generated successfully. Ready for display in your IDE.,49ff50e0-6f09-4e2b-9cc1-1135e4bfe4df,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""undefined""); -",116,0.9869 +",116,0.9869 2025-09-22T14:00:25.252771,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -44733,7 +44733,7 @@ function get_max_triples(n){ } ",86.8216000017128,,Completions generated successfully. Ready for display in your IDE.,c78b7ff5-543f-44e9-b07f-e35d57cab191,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:25.342139,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -44771,7 +44771,7 @@ function is_prime(n){ } ",89.26110000174958,,Completions generated successfully. Ready for display in your IDE.,bd2de108-8840-4be7-900b-3efae35e545d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:25.433000,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44797,7 +44797,7 @@ class Problem { } return false;",},91.19749999808846,,Completions generated successfully. Ready for display in your IDE.,451fb06a-ba11-42b0-b7dc-da900dd4bd2b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:25.498532,[1],"//Create a function that returns true if the last character // of a given string is an alphabetical character and is not // a part of a word, and false otherwise. @@ -44817,7 +44817,7 @@ function check_if_last_char_is_a_letter(txt){ return true; } return false;",,65.06429999717511,,Completions generated successfully. Ready for display in your IDE.,1109df48-a5ff-4388-a02d-a7b772119cbc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:25.587955,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44849,7 +44849,7 @@ class Problem { } return sb.reverse().toString();",},88.86019999772543,,Completions generated successfully. Ready for display in your IDE.,21a75232-84e7-410e-a98d-b276fd2c80d5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:25.744396,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44889,7 +44889,7 @@ class Problem { });"," } }",156.05150000192225,,Completions generated successfully. Ready for display in your IDE.,962449f9-d0aa-4270-89c9-78724c27d574,1,deepseek-ai/deepseek-coder-1.3b-base," return words.get(0); -",130,1.0 +",130,1.0 2025-09-22T14:00:25.858541,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -44927,7 +44927,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",113.9444000000367,,Completions generated successfully. Ready for display in your IDE.,6fbaab42-346b-4154-b1f4-c4dfcd6d5c44,1,deepseek-ai/deepseek-coder-1.3b-base," odd++; -",89,1.0 +",89,1.0 2025-09-22T14:00:26.001802,[1],"//Given a positive integer n, return the count of the numbers of n-digit // positive integers that start or end with 1. function starts_one_ends(n){ @@ -44938,7 +44938,7 @@ function starts_one_ends(n){ return ones.length; }",142.60530000319704,,Completions generated successfully. Ready for display in your IDE.,1e2fef4c-05c6-476e-b074-9856863edfff,1,deepseek-ai/deepseek-coder-1.3b-base," ones.push(i) -",105,1.0 +",105,1.0 2025-09-22T14:00:26.275870,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -44960,7 +44960,7 @@ function find_closest_elements(numbers){ } ",274.6268999981112,,Completions generated successfully. Ready for display in your IDE.,66bdbcee-1cba-4edc-94a2-35636cad3fcf,1,deepseek-ai/deepseek-coder-1.3b-base," for (let j = i + 1; j < numbers.length; j++) { -",248,0.9963 +",248,0.9963 2025-09-22T14:00:26.523708,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -45032,7 +45032,7 @@ function minPath(grid, k){ return path; }",247.12219999491936,,Completions generated successfully. Ready for display in your IDE.,fad15c51-fdf9-4a15-bb1c-5e50e3f7c28d,1,deepseek-ai/deepseek-coder-1.3b-base," neighbours.push(grid[x][y - 1]); -",218,1.0 +",218,1.0 2025-09-22T14:00:26.597688,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -45061,7 +45061,7 @@ function will_it_fly(q, w){ } } return balanced && sum <= w;",,74.01219999883324,,Completions generated successfully. Ready for display in your IDE.,5ebb9fc7-f68f-4715-bd51-79ffe3a7d9eb,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:26.808849,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -45087,7 +45087,7 @@ function string_xor(a, b){ return result; }",211.03159999620402,,Completions generated successfully. Ready for display in your IDE.,83784419-3500-4aeb-aacc-91667815a7ab,1,deepseek-ai/deepseek-coder-1.3b-base," arrayB.push(parseInt(b[j])); -",185,1.0 +",185,1.0 2025-09-22T14:00:26.858636,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45115,7 +45115,7 @@ class Problem { if (n == 3) return 0; return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4); } -}",49.71369999839226,,Completions generated successfully. Ready for display in your IDE.,7dc18ff4-228b-4c6e-b876-5d95523efa0a,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 +}",49.71369999839226,,Completions generated successfully. Ready for display in your IDE.,7dc18ff4-228b-4c6e-b876-5d95523efa0a,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 2025-09-22T14:00:26.979353,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -45143,7 +45143,7 @@ function get_max_triples(n){ }","} ",120.06259999907343,,Completions generated successfully. Ready for display in your IDE.,e1b1c82d-a1c0-4a9f-948f-23de4a0b0772,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",84,1.0 +",84,1.0 2025-09-22T14:00:27.123818,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -45169,7 +45169,7 @@ function separate_paren_groups(paren_string){ return groups; } ",144.43779999419348,,Completions generated successfully. Ready for display in your IDE.,4aa23145-dbb1-4fb4-9dff-b60fa0d52c92,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",104,0.926 +",104,0.926 2025-09-22T14:00:27.440703,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45188,7 +45188,7 @@ class Problem { }",316.29189999512164,,Completions generated successfully. Ready for display in your IDE.,887d022d-2fd2-41ac-9ef8-8afd7000f464,1,deepseek-ai/deepseek-coder-1.3b-base," return numbers.stream() .map(x -> x / ((max-min)*(-1))); -",290,0.939 +",290,0.939 2025-09-22T14:00:27.539568,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45220,7 +45220,7 @@ class Problem { } }",98.76320000330452,,Completions generated successfully. Ready for display in your IDE.,dc8f7cf6-0976-461e-9e90-13e6cc1bb09b,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",69,1.0 +",69,1.0 2025-09-22T14:00:28.232922,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -45244,7 +45244,7 @@ function car_race_collision(n){ //for loop to fill arrays with random numbers between -5 and +5 inclusive while (!Array.isArray(left_to_right)) { - const randLeftToRight = Math.floor((Math.random() * ((2*6)-3+1)) + ",669,0.8739 + const randLeftToRight = Math.floor((Math.random() * ((2*6)-3+1)) + ",669,0.8739 2025-09-22T14:00:28.372122,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45277,7 +45277,7 @@ class Problem { } }",138.85529999970458,,Completions generated successfully. Ready for display in your IDE.,9fe752e0-c021-42f8-bc5f-020be1af6e7a,1,deepseek-ai/deepseek-coder-1.3b-base," long v = 0l; -",115,0.9907 +",115,0.9907 2025-09-22T14:00:28.631691,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -45321,7 +45321,7 @@ function histogram(test){ } ",259.06369999574963,,Completions generated successfully. Ready for display in your IDE.,d7f28ce7-19d8-4f63-a5d5-915d653088f7,1,deepseek-ai/deepseek-coder-1.3b-base," else if ((count[key]) === max && !result[key] ) { -",232,0.9359 +",232,0.9359 2025-09-22T14:00:28.819533,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -45348,7 +45348,7 @@ function next_smallest(lst){ return second_smallest; }",187.4863000048208,,Completions generated successfully. Ready for display in your IDE.,7d09b0dc-e609-479b-893a-fac4806f980a,1,deepseek-ai/deepseek-coder-1.3b-base," second_smallest=lst[i]; -",166,1.0 +",166,1.0 2025-09-22T14:00:28.940278,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -45404,7 +45404,7 @@ function by_length(arr){ }",120.50830000225687,,Completions generated successfully. Ready for display in your IDE.,e2d1bb59-9b47-4830-9709-65710d9f956f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",89,0.9326 +",89,0.9326 2025-09-22T14:00:29.318778,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45428,7 +45428,7 @@ class Problem { }",378.756499994779,,Completions generated successfully. Ready for display in your IDE.,3e7217d8-4c57-4c9e-b9ea-ea1bb86cf7c9,1,deepseek-ai/deepseek-coder-1.3b-base," while ((a %= b) != 0 && (b %= a) != 0); return a + b; -",341,0.9635 +",341,0.9635 2025-09-22T14:00:29.922606,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -45480,7 +45480,7 @@ function check_dict_case(dict){ //console.log(""key:"" + key); console.log(""value:"" + value); if (!isNaN(Number(value)) || typeof Number === 'function') { -",576,0.9377 +",576,0.9377 2025-09-22T14:00:30.430938,[1],"//Given a string s and a natural number n, you have been tasked to implement // a function that returns an array of all words from string s that contain exactly // n consonants, in order these words appear in the string s. @@ -45509,7 +45509,7 @@ function select_words(s, n){ console.log([...new Set(selected)]); }; -",477,0.9124 +",477,0.9124 2025-09-22T14:00:31.127567,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -45540,7 +45540,7 @@ function triples_sum_to_zero(l){ console.log(""Sum: ""+sum); //console.log('i:'+i,'j:'+j,'k:'+k); if (sum === 0) { -",673,0.9704 +",673,0.9704 2025-09-22T14:00:31.217829,[1],"def is_prime(n: int) -> bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -45564,7 +45564,7 @@ function triples_sum_to_zero(l){ if n % i == 0: return False"," ",89.54990000347607,,Completions generated successfully. Ready for display in your IDE.,00c9419f-5fca-498a-9e19-64545e669abe,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",67,1.0 + return True",67,1.0 2025-09-22T14:00:31.313985,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -45589,7 +45589,7 @@ function sum_squares(lst){ return sum; }",96.01579999434762,,Completions generated successfully. Ready for display in your IDE.,7335f2c6-b189-404f-afa5-38ded83abd10,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T14:00:31.398462,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45609,7 +45609,7 @@ class Problem { } return c.toString();",},84.72120000078576,,Completions generated successfully. Ready for display in your IDE.,b835eec8-6006-46c0-9761-62cd74e3bc91,1,deepseek-ai/deepseek-coder-1.3b-base," } -",59,1.0 +",59,1.0 2025-09-22T14:00:31.524996,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45647,7 +45647,7 @@ class Problem { } }",126.13910000072792,,Completions generated successfully. Ready for display in your IDE.,11dbf374-3250-45ef-93a2-c2b8ba7ba9b1,1,deepseek-ai/deepseek-coder-1.3b-base," return lst; -",99,1.0 +",99,1.0 2025-09-22T14:00:31.640590,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -45680,7 +45680,7 @@ class Problem { ",115.4750999994576,,Completions generated successfully. Ready for display in your IDE.,31e99ef6-9e56-4927-8201-81834f7d9f3f,1,deepseek-ai/deepseek-coder-1.3b-base," i = 0 -",87,1.0 +",87,1.0 2025-09-22T14:00:31.862425,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -45747,7 +45747,7 @@ function numerical_letter_grade(grades){ return grades_array; }",221.85759999410948,,Completions generated successfully. Ready for display in your IDE.,5af5a80d-31da-41c8-9476-efbc968716f2,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 3.0){ -",191,1.0 +",191,1.0 2025-09-22T14:00:32.227427,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45782,7 +45782,7 @@ class Problem { } }",364.230300001509,,Completions generated successfully. Ready for display in your IDE.,f61ecebf-9ff0-45b5-acbf-5ffcf1b0b3c2,1,deepseek-ai/deepseek-coder-1.3b-base," lst.set(i, lst.get(i) * lst.get(i)); -",285,1.0 +",285,1.0 2025-09-22T14:00:32.326370,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45824,7 +45824,7 @@ class Problem { if (c1 == 0 || c2 == 0) return ""Yes""; else return ""No"";",},98.37230000266572,,Completions generated successfully. Ready for display in your IDE.,c4165945-f017-4554-bf5a-8f9a396fe0e6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:32.395964,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -45845,7 +45845,7 @@ function can_arrange(arr){ } } return res;",,69.73949999519391,,Completions generated successfully. Ready for display in your IDE.,e7a2d874-6bcb-4394-a8e9-8509f6223118,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T14:00:32.483569,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45885,7 +45885,7 @@ class Problem { } }",87.10040000005392,,Completions generated successfully. Ready for display in your IDE.,2c230020-0dbf-430f-8bcd-6d90a3763ccc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:32.575140,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -45908,7 +45908,7 @@ function odd_count(lst){ return new_lst; }",91.52089999406599,,Completions generated successfully. Ready for display in your IDE.,38430288-c8a1-4588-8893-313ed92929d6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:32.743028,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -45931,7 +45931,7 @@ function odd_count(lst){ return False ",167.70069999620318,,Completions generated successfully. Ready for display in your IDE.,ff71166e-d528-41cc-848f-392919950e8b,1,deepseek-ai/deepseek-coder-1.3b-base," if len(a) == len(b): -",144,0.9307 +",144,0.9307 2025-09-22T14:00:32.859510,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -45961,7 +45961,7 @@ def count_nums(arr: List[int]) -> int: return n ",116.15950000123121,,Completions generated successfully. Ready for display in your IDE.,810ecd7c-38e3-4bb1-aa86-f5b883b70eb4,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T14:00:32.921940,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -45979,7 +45979,7 @@ function count_upper(s){ } return result; }",62.22880000132136,,Completions generated successfully. Ready for display in your IDE.,1eb609ed-53f0-43dd-9307-ae77ce392d31,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:33.008642,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -46046,7 +46046,7 @@ function do_algebra(operator, operand){ return answer; } ",86.47170000040205,,Completions generated successfully. Ready for display in your IDE.,b84bd900-5a7a-4d55-a593-baf31bb5e90b,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:00:33.177755,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46079,7 +46079,7 @@ class Problem { } }",168.58399999910034,,Completions generated successfully. Ready for display in your IDE.,75596307-40fc-4e13-af42-a3fc67f42dce,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(chars); -",135,1.0 +",135,1.0 2025-09-22T14:00:33.875732,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -46106,7 +46106,7 @@ function reverse_delete(s, c){ console.log({arr}); function deleteCharacters(charToDelete:string):void{ - while ((index = arr.indexOf(charToDelete)) !== -1 )",673,0.9245 + while ((index = arr.indexOf(charToDelete)) !== -1 )",673,0.9245 2025-09-22T14:00:33.969873,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -46150,7 +46150,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",93.50500000437023,,Completions generated successfully. Ready for display in your IDE.,51e07160-8886-49ef-98d5-6571edbfe326,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:34.095592,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -46162,7 +46162,7 @@ var b = [""ab"", ""a"", ""aaa"", ""cd""];",93.50500000437023,,Completions genera ",125.92379999841796,,Completions generated successfully. Ready for display in your IDE.,58ba46db-0447-4a40-963f-a3e78bef86fb,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:34.277251,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -46190,7 +46190,7 @@ function get_max_triples(n){ } ",180.95249999896623,,Completions generated successfully. Ready for display in your IDE.,1ee19b15-bc99-4580-b580-9cabb0f5ff08,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T14:00:34.403730,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46215,7 +46215,7 @@ class Problem { return ret; } }",126.77219999750378,,Completions generated successfully. Ready for display in your IDE.,376d1447-e204-49c8-a128-d2bad23b00df,1,deepseek-ai/deepseek-coder-1.3b-base," long ret = 0L; -",102,1.0 +",102,1.0 2025-09-22T14:00:34.561569,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -46229,7 +46229,7 @@ function filter_by_substring(strings, substring){ return filtered; }",157.45260000403505,,Completions generated successfully. Ready for display in your IDE.,14ef5d9e-6bd8-4596-8024-0915bef9bddf,1,deepseek-ai/deepseek-coder-1.3b-base," filtered.push(strings[i]); -",135,1.0 +",135,1.0 2025-09-22T14:00:34.645046,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -46240,7 +46240,7 @@ function all_prefixes(string){ } ",83.52270000614226,,Completions generated successfully. Ready for display in your IDE.,1b62a3b1-8ace-4c97-b9f0-588a0c5dd953,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:34.713379,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46280,7 +46280,7 @@ class Problem { } return arr.size() == 3; }",,67.72890000138432,,Completions generated successfully. Ready for display in your IDE.,187774ef-26a1-4125-bffd-72f2081e0c37,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T14:00:34.784977,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46306,7 +46306,7 @@ class Problem { } return levels; }",,71.76649999746587,,Completions generated successfully. Ready for display in your IDE.,012fa140-8d2f-48c3-a3c0-c0eb0dea62a6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:34.828571,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -46329,7 +46329,7 @@ function parse_music(music_string){ }); return song; } -",43.47630000120262,,Completions generated successfully. Ready for display in your IDE.,3451a599-1231-438f-b666-67ce7bd4b697,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.47630000120262,,Completions generated successfully. Ready for display in your IDE.,3451a599-1231-438f-b666-67ce7bd4b697,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:00:34.940514,[1],"from typing import List def f(n: int) -> List[int]: @@ -46351,7 +46351,7 @@ def f(n: int) -> List[int]: return a pass",111.51740000059363,,Completions generated successfully. Ready for display in your IDE.,4f159682-ba6d-4969-b0a0-2571aee76719,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",78,1.0 +",78,1.0 2025-09-22T14:00:35.048025,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -46378,7 +46378,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: if any(not x.islower() for x in dict.keys()): if any(not x.isupper() for x in dict.keys()): return False",,106.94769999827258,,Completions generated successfully. Ready for display in your IDE.,ce53c629-a262-4ca5-8551-3ca0a49a4892,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",63,0.989 + return True",63,0.989 2025-09-22T14:00:35.742963,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -46414,7 +46414,7 @@ function words_in_sentence(sentence){ function getPrimes (arrOfNumbers) { return arrOfNumbers.filter((item) =>{ - item > 3 && Number.",669,0.8763 + item > 3 && Number.",669,0.8763 2025-09-22T14:00:35.884942,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -46439,7 +46439,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",141.24509999965085,,Completions generated successfully. Ready for display in your IDE.,972912ff-552c-4fb3-9a67-81c6dee0b2d3,1,deepseek-ai/deepseek-coder-1.3b-base," if c == '(': -",117,1.0 +",117,1.0 2025-09-22T14:00:36.011941,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46490,7 +46490,7 @@ class Problem { } }",127.00939999922412,,Completions generated successfully. Ready for display in your IDE.,0de6dfb0-61e2-4ecd-9f0b-e73604b138dd,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",100,1.0 +",100,1.0 2025-09-22T14:00:36.274931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46525,7 +46525,7 @@ class Problem { } }",262.47320000402397,,Completions generated successfully. Ready for display in your IDE.,e8c7e93b-ef9e-436a-ae60-b57f5ae12f86,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sz; i++) { -",222,1.0 +",222,1.0 2025-09-22T14:00:36.368933,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46541,7 +46541,7 @@ class Problem { TreeSet tree = new TreeSet(l); return new ArrayList(tree);",},93.97679999528918,,Completions generated successfully. Ready for display in your IDE.,aeb4ca78-91fb-4aa0-a25c-f623f82c3aab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:36.439931,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -46556,7 +46556,7 @@ function rolling_max(numbers){ max.push(tempMax); } return max;",,71.4631999944686,,Completions generated successfully. Ready for display in your IDE.,58b3e386-4081-4c09-9277-537b0bc3cd05,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:36.627929,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -46623,7 +46623,7 @@ function numerical_letter_grade(grades){ return grades_array; }",187.60330000077374,,Completions generated successfully. Ready for display in your IDE.,b82af78c-aa2e-4b77-ad69-b69bbee5a759,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""D-""); -",160,1.0 +",160,1.0 2025-09-22T14:00:36.720933,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46656,7 +46656,7 @@ class Problem { } return ret;",},92.64729999995325,,Completions generated successfully. Ready for display in your IDE.,f3e6b443-1bcf-456b-bddd-e20af46f4db1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:36.832930,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -46677,7 +46677,7 @@ class Problem { ",111.18849999911617,,Completions generated successfully. Ready for display in your IDE.,7fba0db8-cb80-4664-ac62-0ab96e41c2d3,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",80,1.0 +",80,1.0 2025-09-22T14:00:36.903928,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -46709,7 +46709,7 @@ class Problem { return count ",70.50370000069961,,Completions generated successfully. Ready for display in your IDE.,d24cdee8-e371-4ea9-a912-cb25ff45d415,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T14:00:37.051763,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46751,7 +46751,7 @@ class Problem { } }",148.1701999946381,,Completions generated successfully. Ready for display in your IDE.,4a4670e5-950a-4ad1-a0e5-5ae31c69a77d,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(s); -",120,1.0 +",120,1.0 2025-09-22T14:00:37.211812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46785,7 +46785,7 @@ class Problem { } }",159.36830000282498,,Completions generated successfully. Ready for display in your IDE.,c9ff9976-8690-49a8-b40a-2ddb44f95d9f,1,deepseek-ai/deepseek-coder-1.3b-base," if (c == '<') { -",135,1.0 +",135,1.0 2025-09-22T14:00:37.296760,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46808,7 +46808,7 @@ class Problem { } }",84.90059999894584,,Completions generated successfully. Ready for display in your IDE.,08bb920f-8a0d-4a0e-aefb-fe590d5ae615,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:37.495762,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -46829,7 +46829,7 @@ function specialFilter(nums){ } ",198.1524000002537,,Completions generated successfully. Ready for display in your IDE.,c1a92bd5-850d-48b2-bc0b-9755f4e0968e,1,deepseek-ai/deepseek-coder-1.3b-base," const num = Math.abs(nums[i]); -",174,0.9757 +",174,0.9757 2025-09-22T14:00:37.601760,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -46852,7 +46852,7 @@ function specialFilter(nums){ return count ",105.57460000563879,,Completions generated successfully. Ready for display in your IDE.,47685c9f-b20c-4891-b98c-29372d65c707,1,deepseek-ai/deepseek-coder-1.3b-base," i = 0 -",82,1.0 +",82,1.0 2025-09-22T14:00:37.727760,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -46868,7 +46868,7 @@ function specialFilter(nums){ d, m = divmod(n, 10) return digits(d) * (m if m % 2 == 1 else 1) ",125.85050000052433,,Completions generated successfully. Ready for display in your IDE.,48145bb8-aded-4a31-9bf3-7bd4282fde4c,1,deepseek-ai/deepseek-coder-1.3b-base," if n < 10: -",102,1.0 +",102,1.0 2025-09-22T14:00:37.854762,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46907,7 +46907,7 @@ class Problem { } }",126.42580000101589,,Completions generated successfully. Ready for display in your IDE.,d93fe5d5-d219-463a-92f2-dc1cada56247,1,deepseek-ai/deepseek-coder-1.3b-base," index++; -",94,1.0 +",94,1.0 2025-09-22T14:00:37.948758,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46947,7 +46947,7 @@ class Problem { } }",94.26100000564475,,Completions generated successfully. Ready for display in your IDE.,552dbc6b-5eb8-4eef-80ea-fcf11d7e78a0,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",68,1.0 +",68,1.0 2025-09-22T14:00:38.213638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46980,7 +46980,7 @@ class Problem { } }",264.2120000018622,,Completions generated successfully. Ready for display in your IDE.,e00e4000-af95-454e-b314-c39ce0096dd5,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (brackets.charAt(i) == ')') { -",225,0.954 +",225,0.954 2025-09-22T14:00:38.308426,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -47021,7 +47021,7 @@ function int_to_mini_roman(number){ } ",95.15259999898262,,Completions generated successfully. Ready for display in your IDE.,0b830f3f-741c-4632-b43a-26061a449f35,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T14:00:38.422023,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -47051,7 +47051,7 @@ function closest_integer(value){ } ",113.19139999977779,,Completions generated successfully. Ready for display in your IDE.,b8b816eb-9882-486e-90cd-6c21bd433de5,1,deepseek-ai/deepseek-coder-1.3b-base," }else{ -",82,1.0 +",82,1.0 2025-09-22T14:00:38.549484,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -47087,7 +47087,7 @@ function closest_integer(value){ ",127.40959999791812,,Completions generated successfully. Ready for display in your IDE.,124740c5-4b13-48ad-8f85-1a533b6ce03e,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",100,1.0 +",100,1.0 2025-09-22T14:00:38.635765,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -47123,7 +47123,7 @@ function move_one_ball(arr){ }"," return true; } }",85.96440000110306,,Completions generated successfully. Ready for display in your IDE.,1a07abae-3143-457e-b09e-e98f54696256,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",59,1.0 +",59,1.0 2025-09-22T14:00:38.698005,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -47164,7 +47164,7 @@ function is_sorted(lst){ return true; } ",62.118700006976724,,Completions generated successfully. Ready for display in your IDE.,47236fc0-c0d7-4da9-95c0-a018990ae555,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:00:39.067791,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47197,7 +47197,7 @@ class Problem { }",369.85850000201026,,Completions generated successfully. Ready for display in your IDE.,e6c8cc04-baa6-4208-aa22-a3e18cb6a68e,1,deepseek-ai/deepseek-coder-1.3b-base," long value = hashmap.get(key).longValue() + 1; hashmap.put(key, value); -",336,1.0 +",336,1.0 2025-09-22T14:00:39.314109,[1],"def count_distinct_characters(string: str) -> int: """""" Given a string, find out how many distinct characters (regardless of case) does it consist of >>> count_distinct_characters('xyzXYZ') @@ -47212,7 +47212,7 @@ class Problem { ",245.9409000002779,,Completions generated successfully. Ready for display in your IDE.,4075ae68-d8de-4fad-82a5-bb09220624d9,1,deepseek-ai/deepseek-coder-1.3b-base," if not l.isspace(): s.add(l.lower()) -",225,0.9705 +",225,0.9705 2025-09-22T14:00:39.555574,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -47239,7 +47239,7 @@ def sum_squares(lst: List[float]) -> int: total += i**2 return total",241.06019999453565,,Completions generated successfully. Ready for display in your IDE.,c3da28fa-6994-4f04-9322-980749254ec7,1,deepseek-ai/deepseek-coder-1.3b-base," lst[i] = math.ceil(lst[i]) -",206,1.0 +",206,1.0 2025-09-22T14:00:39.686749,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47270,7 +47270,7 @@ class Problem { } }",131.01879999885568,,Completions generated successfully. Ready for display in your IDE.,f2b2dac3-fc7f-4e70-a865-3313ce15782f,1,deepseek-ai/deepseek-coder-1.3b-base," boolean dec = true; -",104,1.0 +",104,1.0 2025-09-22T14:00:39.777994,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -47289,7 +47289,7 @@ function count_upper(s){ return result; }",90.30300000449643,,Completions generated successfully. Ready for display in your IDE.,12d87598-58a7-49c4-a95b-c4628d05b569,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:39.958781,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -47323,7 +47323,7 @@ function count_upper(s){ for word in words:"," prime_words.append(word) return ' '.join(prime_words)",181.08989999745972,,Completions generated successfully. Ready for display in your IDE.,951455f2-aff0-4df3-912a-35f49291fcfd,1,deepseek-ai/deepseek-coder-1.3b-base," if is_prime(len(word)): -",156,1.0 +",156,1.0 2025-09-22T14:00:40.161201,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47365,7 +47365,7 @@ class Problem { } }",202.14679999480722,,Completions generated successfully. Ready for display in your IDE.,d9e3b8bd-d48c-4025-98e8-86c64017b96e,1,deepseek-ai/deepseek-coder-1.3b-base," String s4 = s2+s1; -",162,1.0 +",162,1.0 2025-09-22T14:00:40.227529,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -47392,7 +47392,7 @@ function fib4(n){ prev[3] = curr; } return prev[3];",,66.11839999823133,,Completions generated successfully. Ready for display in your IDE.,48f0ea37-c830-419f-9d0b-d0b259f40103,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T14:00:40.321624,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -47434,7 +47434,7 @@ function is_sorted(lst){ } ",93.78899999865098,,Completions generated successfully. Ready for display in your IDE.,e5903a22-d900-4132-8c74-ffad8a70ec56,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:40.387448,[1],"//Create a function encrypt that takes a string as an argument and // returns a string encrypted with the alphabet being rotated. // The alphabet should be rotated in a manner such that the letters @@ -47457,7 +47457,7 @@ function encrypt(s){ newString += alphabet[(index + shiftBy) % 26]; } return newString;",,65.35119999898598,,Completions generated successfully. Ready for display in your IDE.,011dfd97-db5d-41c1-a8ff-daa292f27c41,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:40.602219,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47499,7 +47499,7 @@ class Problem { } }",214.93599999666912,,Completions generated successfully. Ready for display in your IDE.,88148c46-6f14-40a5-9890-80b5ec49dff7,1,deepseek-ai/deepseek-coder-1.3b-base," int diff = Math.abs(cap-sm); -",174,0.9759 +",174,0.9759 2025-09-22T14:00:40.660242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47530,7 +47530,7 @@ class Problem { return result; } }",57.99709999701008,,Completions generated successfully. Ready for display in your IDE.,b4466039-2a63-4578-9c09-f89c668ee450,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T14:00:40.747866,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -47549,7 +47549,7 @@ function is_bored(S){ } ",86.74789999349741,,Completions generated successfully. Ready for display in your IDE.,1b49715e-3d96-45ae-8932-491addb14f98,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:40.853990,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -47577,7 +47577,7 @@ function is_bored(S){ ",105.8878999974695,,Completions generated successfully. Ready for display in your IDE.,4bd4cc5b-5717-4597-a0ec-670743488922,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T14:00:40.951782,[1],"def is_simple_power(x: int, n: int) -> bool: """"""Your task is to write a function that returns true if a number x is a simple power of n and false in other cases. @@ -47604,7 +47604,7 @@ function is_bored(S){ return False x /= n"," ",98.12160000001313,,Completions generated successfully. Ready for display in your IDE.,5bd96b51-10ef-4985-9830-66372e4d529d,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",63,0.9364 + return True",63,0.9364 2025-09-22T14:00:41.150730,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -47630,7 +47630,7 @@ function is_bored(S){ return '' ",198.52030000038212,,Completions generated successfully. Ready for display in your IDE.,dc19b8cc-98fa-47d2-8ea5-d42c5cd2a72e,1,deepseek-ai/deepseek-coder-1.3b-base," for i, c in enumerate(word): -",155,0.9588 +",155,0.9588 2025-09-22T14:00:41.399518,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47695,7 +47695,7 @@ class Problem { } }",249.12890000268817,,Completions generated successfully. Ready for display in your IDE.,cb12ef77-4694-448c-8c4e-2cad6031a3ee,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 0.7f) { -",216,1.0 +",216,1.0 2025-09-22T14:00:41.562517,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47728,7 +47728,7 @@ class Problem { return ret; } }",161.85489999770653,,Completions generated successfully. Ready for display in your IDE.,aa7286a5-7508-45f4-9d0a-bd9029acc134,1,deepseek-ai/deepseek-coder-1.3b-base," var ret = new ArrayList(); -",136,0.9315 +",136,0.9315 2025-09-22T14:00:41.741722,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47759,7 +47759,7 @@ class Problem { return (res_num % res_denom == 0); } }",179.21799999749055,,Completions generated successfully. Ready for display in your IDE.,cfff13f5-bcaf-4ab7-b958-5dcc271f3cb6,1,deepseek-ai/deepseek-coder-1.3b-base," String[] nums = x.split(""/""); -",152,1.0 +",152,1.0 2025-09-22T14:00:41.817199,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47809,7 +47809,7 @@ class Problem { return ""Yes""; } }",75.18349999736529,,Completions generated successfully. Ready for display in your IDE.,70f6cbe1-3162-4d09-824e-a4ebf3cb020a,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:00:41.955541,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -47847,7 +47847,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",138.46210000338033,,Completions generated successfully. Ready for display in your IDE.,6ba1eaa1-9666-45dc-9d3d-dd88daa45063,1,deepseek-ai/deepseek-coder-1.3b-base," var even = 0; -",110,1.0 +",110,1.0 2025-09-22T14:00:42.095528,[1],"def is_bored(S: str) -> int: """""" You'll be given a string of words, and your task is to count the number @@ -47868,7 +47868,7 @@ function even_odd_palindrome(n){ ",140.11000000027707,,Completions generated successfully. Ready for display in your IDE.,d28aea95-0125-4009-9c3c-95872dd1a39b,1,deepseek-ai/deepseek-coder-1.3b-base," for sentence in sentences: -",98,1.0 +",98,1.0 2025-09-22T14:00:42.213722,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -47891,7 +47891,7 @@ function even_odd_palindrome(n){ return n * special_factorial(n-1) * factorial(n-1) ",117.95970000093803,,Completions generated successfully. Ready for display in your IDE.,e36291f1-65cb-4977-b36f-75d5cef3cee0,1,deepseek-ai/deepseek-coder-1.3b-base," def factorial(n): -",95,0.9845 +",95,0.9845 2025-09-22T14:00:42.267610,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -47926,7 +47926,7 @@ function max_fill(grid, capacity){ } maxFill += Math.ceil(sum/capacity); } - return maxFill;",,53.69529999734368,,Completions generated successfully. Ready for display in your IDE.,d6600fb2-017e-4cc5-bd84-060bb621d6f3,1,deepseek-ai/deepseek-coder-1.3b-base,,26,0.8386 + return maxFill;",,53.69529999734368,,Completions generated successfully. Ready for display in your IDE.,d6600fb2-017e-4cc5-bd84-060bb621d6f3,1,deepseek-ai/deepseek-coder-1.3b-base,,26,0.8386 2025-09-22T14:00:42.553222,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -47948,7 +47948,7 @@ function double_the_difference(lst){ ",285.65530000196304,,Completions generated successfully. Ready for display in your IDE.,9d19388a-e789-4600-9989-ddb44536b311,1,deepseek-ai/deepseek-coder-1.3b-base," var num = Math.pow(lst[i], 2); sum += num; -",262,0.9362 +",262,0.9362 2025-09-22T14:00:42.629319,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -48015,7 +48015,7 @@ function do_algebra(operator, operand){ operator.splice(0, 1); } return answer;",,75.34660000237636,,Completions generated successfully. Ready for display in your IDE.,9c831676-8cc1-4a24-81c7-e6ccdd01d26e,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,0.9321 +}",50,0.9321 2025-09-22T14:00:42.716684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48062,7 +48062,7 @@ class Problem { } return count; }",,87.65249999851221,,Completions generated successfully. Ready for display in your IDE.,4b927543-768d-4b50-8954-a21a682e7854,1,deepseek-ai/deepseek-coder-1.3b-base," -}",61,1.0 +}",61,1.0 2025-09-22T14:00:42.808018,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -48101,7 +48101,7 @@ function match_parens(lst){ } return 'No'; }",90.80959999846527,,Completions generated successfully. Ready for display in your IDE.,2c82dc16-9c42-435f-b003-adf5f43d1fe5,1,deepseek-ai/deepseek-coder-1.3b-base," - } ",67,0.6931 + } ",67,0.6931 2025-09-22T14:00:42.965122,[1],"from typing import List def incr_list(l: List[int]) -> List[int]: @@ -48115,7 +48115,7 @@ def incr_list(l: List[int]) -> List[int]: for i in l:"," return res ",156.29579999949783,,Completions generated successfully. Ready for display in your IDE.,856defa8-890b-4972-9feb-8eeed8d84866,1,deepseek-ai/deepseek-coder-1.3b-base," res.append(i + 1) -",134,0.9814 +",134,0.9814 2025-09-22T14:00:43.169983,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48177,7 +48177,7 @@ class Problem { } }",204.75080000323942,,Completions generated successfully. Ready for display in your IDE.,28aff54b-8d37-41c3-9f2b-d8cf564803d1,1,deepseek-ai/deepseek-coder-1.3b-base," largestPrime /= 10; -",163,1.0 +",163,1.0 2025-09-22T14:00:43.277547,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -48199,7 +48199,7 @@ function modp(n, p){ ",107.5732999961474,,Completions generated successfully. Ready for display in your IDE.,915a4eab-1019-4c81-a092-d5b509948734,1,deepseek-ai/deepseek-coder-1.3b-base," var result = 1; -",84,1.0 +",84,1.0 2025-09-22T14:00:43.547155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48241,7 +48241,7 @@ class Problem { } }",269.63119999709306,,Completions generated successfully. Ready for display in your IDE.,dcbf730f-3b28-4639-890c-8b10bc268bda,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append(Character.toUpperCase(s.charAt(i))); -",244,1.0 +",244,1.0 2025-09-22T14:00:43.731131,[1],"def greatest_common_divisor(a: int, b: int) -> int: """""" Return a greatest common divisor of two integers a and b >>> greatest_common_divisor(3, 5) @@ -48255,7 +48255,7 @@ class Problem { ",183.35970000043744,,Completions generated successfully. Ready for display in your IDE.,9065323f-1ad2-4b6c-8632-246e5b8a7fb8,1,deepseek-ai/deepseek-coder-1.3b-base," while (b % a) != 0: -",161,0.9577 +",161,0.9577 2025-09-22T14:00:43.920092,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48300,7 +48300,7 @@ class Problem { } }",188.50120000570314,,Completions generated successfully. Ready for display in your IDE.,925b3e3a-725a-4f6a-aaae-dd263f3fbe66,1,deepseek-ai/deepseek-coder-1.3b-base," if (word.length() == prime) { -",161,1.0 +",161,1.0 2025-09-22T14:00:44.013106,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -48338,7 +48338,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",93.56629999820143,,Completions generated successfully. Ready for display in your IDE.,33942b97-f24b-48a1-b3d2-342ef8933d02,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:44.150091,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -48381,7 +48381,7 @@ function get_odd_collatz(n){ } } ",136.40469999518245,,Completions generated successfully. Ready for display in your IDE.,441c11f3-a3f1-4a9e-906b-1abc24b3edc6,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",111,0.9215 +",111,0.9215 2025-09-22T14:00:44.338095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48459,7 +48459,7 @@ class Problem { } }",187.21029999869643,,Completions generated successfully. Ready for display in your IDE.,5034ccb1-8efa-4945-956a-5d3f074cea2f,1,deepseek-ai/deepseek-coder-1.3b-base," while (num >= 100) { -",156,1.0 +",156,1.0 2025-09-22T14:00:45.046092,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48500,7 +48500,7 @@ class Problem { if(!result.isEmpty()) { long val = result.remove(result.size()-1); System.out.println(""val "" + val); - for (;",678,0.945 + for (;",678,0.945 2025-09-22T14:00:45.214097,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -48555,7 +48555,7 @@ function by_length(arr){ return arr3; }",167.05799999908777,,Completions generated successfully. Ready for display in your IDE.,f9fc0c92-3526-44b6-89ee-ff9680d16d71,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Nine""); -",140,1.0 +",140,1.0 2025-09-22T14:00:45.310090,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48587,7 +48587,7 @@ class Problem { } }",95.87879999889992,,Completions generated successfully. Ready for display in your IDE.,46e53292-7256-46b4-bf1a-93cec1350a59,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:45.438091,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48627,7 +48627,7 @@ class Problem { } }",127.85339999390999,,Completions generated successfully. Ready for display in your IDE.,232c1b76-1cb9-4082-8c94-b0dfc18a201e,1,deepseek-ai/deepseek-coder-1.3b-base," unique = false; -",98,1.0 +",98,1.0 2025-09-22T14:00:45.522090,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -48654,7 +48654,7 @@ def sum_squares(lst: List[float]) -> int: for i in lst: total += i**2 return total",83.70209999702638,,Completions generated successfully. Ready for display in your IDE.,bf17f12f-7088-4d24-bc75-62953a3adb9b,1,deepseek-ai/deepseek-coder-1.3b-base," import math -",62,0.9378 +",62,0.9378 2025-09-22T14:00:45.779548,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -48669,7 +48669,7 @@ function flip_case(string){ } ",256.7272999949637,,Completions generated successfully. Ready for display in your IDE.,a6e3beba-7510-4296-8e2a-1a788c3f9796,1,deepseek-ai/deepseek-coder-1.3b-base," if (string[i] == string[i].toUpperCase()){ -",227,0.9369 +",227,0.9369 2025-09-22T14:00:45.863216,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -48696,7 +48696,7 @@ function file_name_check(file_name){ ",83.58710000175051,,Completions generated successfully. Ready for display in your IDE.,b8cc9e21-4c38-4815-90d6-6a777596ebbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",58,1.0 +",58,1.0 2025-09-22T14:00:45.958693,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48736,7 +48736,7 @@ class Problem { } }",95.6951999978628,,Completions generated successfully. Ready for display in your IDE.,45d9e4b2-f242-4893-9ac0-3e56cb8b99e2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:46.061284,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -48785,7 +48785,7 @@ function check_dict_case(dict){ return true; }",102.36969999823486,,Completions generated successfully. Ready for display in your IDE.,6474c302-699b-4960-83fc-ed44b01d86fb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:00:46.169350,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48815,7 +48815,7 @@ class Problem { factors.add(n);"," } }",107.25719999754801,,Completions generated successfully. Ready for display in your IDE.,d907c9d9-e8c2-4aca-8ce6-02c4bb17756a,1,deepseek-ai/deepseek-coder-1.3b-base," return factors; -",83,1.0 +",83,1.0 2025-09-22T14:00:46.336490,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48841,7 +48841,7 @@ class Problem { } }",167.15710000426043,,Completions generated successfully. Ready for display in your IDE.,f62f098d-8cec-437b-9ad3-f308c2a56f72,1,deepseek-ai/deepseek-coder-1.3b-base," sum += lst.get(i); -",144,1.0 +",144,1.0 2025-09-22T14:00:46.412889,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -48867,7 +48867,7 @@ function correct_bracketing(brackets){ } ",76.31169999513077,,Completions generated successfully. Ready for display in your IDE.,c8b1575b-b4e4-48a5-886a-bbfd7ddc9a47,1,deepseek-ai/deepseek-coder-1.3b-base," } -",55,1.0 +",55,1.0 2025-09-22T14:00:46.535218,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -48895,7 +48895,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: else: return ""YES""",122.27100000018254,,Completions generated successfully. Ready for display in your IDE.,d1330a03-9eb2-456e-b793-efe097f14e73,1,deepseek-ai/deepseek-coder-1.3b-base," if count > 0: -",99,0.9956 +",99,0.9956 2025-09-22T14:00:46.623383,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48924,7 +48924,7 @@ class Problem { }); return String.join("" "", split);",},88.00029999838443,,Completions generated successfully. Ready for display in your IDE.,9325f95a-1127-469a-9120-acf465cd698e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:46.700432,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -48947,7 +48947,7 @@ function factorize(n){ } ",76.7615000004298,,Completions generated successfully. Ready for display in your IDE.,9c7a0bc4-aeb4-4d53-b65f-1340ffd623d5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",55,1.0 +",55,1.0 2025-09-22T14:00:46.842162,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -48977,7 +48977,7 @@ function sort_third(l){ return x; }",141.6962000002968,,Completions generated successfully. Ready for display in your IDE.,ea5fce9f-9e89-4292-8215-9fe35944971d,1,deepseek-ai/deepseek-coder-1.3b-base," k=k+1; -",110,0.9651 +",110,0.9651 2025-09-22T14:00:46.986242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49015,7 +49015,7 @@ class Problem { } }",143.7277000004542,,Completions generated successfully. Ready for display in your IDE.,65f61724-48bb-4e89-8334-ec0487ada77c,1,deepseek-ai/deepseek-coder-1.3b-base," return lst1; -",117,1.0 +",117,1.0 2025-09-22T14:00:47.029848,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -49037,7 +49037,7 @@ class Problem { count += 1 i += 1 return count -",43.295899995428044,,Completions generated successfully. Ready for display in your IDE.,87224c11-f1b5-4d99-b628-0ee9f997920b,1,deepseek-ai/deepseek-coder-1.3b-base,,15,1.0 +",43.295899995428044,,Completions generated successfully. Ready for display in your IDE.,87224c11-f1b5-4d99-b628-0ee9f997920b,1,deepseek-ai/deepseek-coder-1.3b-base,,15,1.0 2025-09-22T14:00:47.094245,[1],"//Concatenate array of strings into a single string // >>> concatenate([]) // """" @@ -49050,7 +49050,7 @@ function concatenate(strings){ } ",64.16429999808315,,Completions generated successfully. Ready for display in your IDE.,96bc36ca-35df-452b-8e9e-cec816354ff4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",43,0.9577 +",43,0.9577 2025-09-22T14:00:47.232561,[1],"from typing import List def rescale_to_unit(numbers: List[float]) -> List[float]: @@ -49063,7 +49063,7 @@ def rescale_to_unit(numbers: List[float]) -> List[float]: return [(val - _min) / (_max - _min) for val in numbers] ",137.70569999906002,,Completions generated successfully. Ready for display in your IDE.,a91b9805-0d45-4ec8-b10c-a7b7da4d175a,1,deepseek-ai/deepseek-coder-1.3b-base," if _min == _max: -",117,0.9537 +",117,0.9537 2025-09-22T14:00:47.431869,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49101,7 +49101,7 @@ class Problem { } }",198.91260000440525,,Completions generated successfully. Ready for display in your IDE.,821e2f8b-cb51-45d6-89fe-61f479b7e0a8,1,deepseek-ai/deepseek-coder-1.3b-base," for (String s : lst1) { -",169,1.0 +",169,1.0 2025-09-22T14:00:47.482590,[1],"from typing import List def count_up_to(n: int) -> List[int]: @@ -49132,7 +49132,7 @@ def count_up_to(n: int) -> List[int]: result.append(number) return result ",50.84749999514315,,Completions generated successfully. Ready for display in your IDE.,46743131-013f-420d-8708-7c115110d127,1,deepseek-ai/deepseek-coder-1.3b-base," -",27,1.0 +",27,1.0 2025-09-22T14:00:47.538321,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -49154,7 +49154,7 @@ function factorize(n){ } } return arr;",,55.37499999627471,,Completions generated successfully. Ready for display in your IDE.,878f856a-0156-4e5a-a20e-c6202b0b7e7d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",35,0.9784 +}",35,0.9784 2025-09-22T14:00:47.663318,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -49202,7 +49202,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",125.0235999978031,,Completions generated successfully. Ready for display in your IDE.,5969c117-2e25-4cba-b830-c068c584578c,1,deepseek-ai/deepseek-coder-1.3b-base," row = i -",97,1.0 +",97,1.0 2025-09-22T14:00:47.750844,[1],"def multiply(a: int, b: int) -> int: """"""Complete the function that takes two integers and returns the product of their unit digits. @@ -49218,7 +49218,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: 20 """""" "," -",87.63560000079451,,Completions generated successfully. Ready for display in your IDE.,b6dae7c7-6c6b-404a-b7da-29626fc98fc6,1,deepseek-ai/deepseek-coder-1.3b-base, return a * b,66,1.0 +",87.63560000079451,,Completions generated successfully. Ready for display in your IDE.,b6dae7c7-6c6b-404a-b7da-29626fc98fc6,1,deepseek-ai/deepseek-coder-1.3b-base, return a * b,66,1.0 2025-09-22T14:00:47.850632,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49267,7 +49267,7 @@ class Problem { } }",99.25929999735672,,Completions generated successfully. Ready for display in your IDE.,3717da84-8b9a-444b-bfb6-d80bdf62659a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:48.335832,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49315,7 +49315,7 @@ class Problem { } } }",484.610799998336,,Completions generated successfully. Ready for display in your IDE.,4c01768d-9fce-432f-a610-fd38de4943b8,1,deepseek-ai/deepseek-coder-1.3b-base," List planets = Arrays.asList(""Mercury"",""Venus"",""Earth"",""Mars"",""Jupiter"",""Saturn"",""Uranus"",""Neptune""); -",456,1.0 +",456,1.0 2025-09-22T14:00:48.410707,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -49342,7 +49342,7 @@ function fib4(n){ return prev[3]; } ",74.42969999829074,,Completions generated successfully. Ready for display in your IDE.,0e5bdb7b-02be-4d97-abec-761a9c0300db,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",51,0.981 +",51,0.981 2025-09-22T14:00:48.474462,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -49363,7 +49363,7 @@ function car_race_collision(n){ } } return num_collisions;",,64.21659999614349,,Completions generated successfully. Ready for display in your IDE.,b3e8fb2b-61b1-4fdd-8b1c-672bf26998e4,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,0.9618 +}",41,0.9618 2025-09-22T14:00:48.728620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49393,7 +49393,7 @@ class Problem { } }",253.37519999447977,,Completions generated successfully. Ready for display in your IDE.,411d3c48-ee77-42e9-9117-39654b08019b,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < text.length(); i++) { -",229,0.9964 +",229,0.9964 2025-09-22T14:00:48.837571,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -49435,7 +49435,7 @@ function compare_one(a, b){ } ",109.03069999767467,,Completions generated successfully. Ready for display in your IDE.,1a76a083-94fb-439e-8375-083687c7b5b2,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",84,1.0 +",84,1.0 2025-09-22T14:00:48.977663,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -49460,7 +49460,7 @@ function sort_numbers(numbers){ return numString; }",139.71190000302158,,Completions generated successfully. Ready for display in your IDE.,1e4ab56c-395a-4912-bfbe-47c9f05ef0e9,1,deepseek-ai/deepseek-coder-1.3b-base," ""nine"": 9 -",115,0.9916 +",115,0.9916 2025-09-22T14:00:49.157296,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -49485,7 +49485,7 @@ def unique_digits(x: List[int]) -> List[int]: temp=temp//10 for i in s: x.remove(i)",,179.9278999969829,,Completions generated successfully. Ready for display in your IDE.,f09f436c-27d6-47ed-9f41-a6f518d4dc0e,1,deepseek-ai/deepseek-coder-1.3b-base," - return sorted(list(set(x)))",145,0.9421 + return sorted(list(set(x)))",145,0.9421 2025-09-22T14:00:49.353736,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -49525,7 +49525,7 @@ function get_closest_vowel(word){ break; }else{ continue; -",170,0.9726 +",170,0.9726 2025-09-22T14:00:49.466731,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49603,7 +49603,7 @@ class Problem { } }",112.08469999837689,,Completions generated successfully. Ready for display in your IDE.,8af8c720-5f76-461f-ba33-206dc0040d49,1,deepseek-ai/deepseek-coder-1.3b-base," } -",78,1.0 +",78,1.0 2025-09-22T14:00:49.552740,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49632,7 +49632,7 @@ class Problem { } }",85.99669999966864,,Completions generated successfully. Ready for display in your IDE.,1fcf8381-4acb-469d-b132-884544fd4892,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:49.686247,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49694,7 +49694,7 @@ class Problem { } }",133.02970000222558,,Completions generated successfully. Ready for display in your IDE.,b953fcf4-33fe-4622-afd9-82d651c69ab8,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",101,0.9689 +",101,0.9689 2025-09-22T14:00:49.783663,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -49749,7 +49749,7 @@ function by_length(arr){ return arr3; }",97.28499999619089,,Completions generated successfully. Ready for display in your IDE.,ecdcf00b-0074-4809-9cee-15d12d4394d3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:49.987684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49814,7 +49814,7 @@ class Problem { } }",204.01649999985239,,Completions generated successfully. Ready for display in your IDE.,0cacfb7f-eadc-449d-8b8f-d26a809de169,1,deepseek-ai/deepseek-coder-1.3b-base," letterGrades.add(""B+""); -",165,1.0 +",165,1.0 2025-09-22T14:00:50.158534,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -49837,7 +49837,7 @@ class Problem { ",170.19230000005336,,Completions generated successfully. Ready for display in your IDE.,8e2ea3a8-f0df-466a-a57b-afceb2326865,1,deepseek-ai/deepseek-coder-1.3b-base," if (b == a[-i:]): -",138,0.9309 +",138,0.9309 2025-09-22T14:00:50.367120,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49867,7 +49867,7 @@ class Problem { } }",208.80619999661576,,Completions generated successfully. Ready for display in your IDE.,334537dc-1e03-4030-9796-858ee18f4856,1,deepseek-ai/deepseek-coder-1.3b-base," newString.append(reverse.substring(1)); -",185,0.9971 +",185,0.9971 2025-09-22T14:00:50.466989,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -49889,7 +49889,7 @@ function cycpattern_check(a, b){ } ",99.34380000049714,,Completions generated successfully. Ready for display in your IDE.,7fa41cb8-0233-4c1c-9fd7-0bd3c92d02e2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:50.775748,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -49901,7 +49901,7 @@ function cycpattern_check(a, b){ ",308.73260000225855,,Completions generated successfully. Ready for display in your IDE.,05f2e04c-a644-47a4-99d7-881f9ae25280,1,deepseek-ai/deepseek-coder-1.3b-base," if str(i)[0] == '1' or str(i)[-1] == '1': -",284,0.9815 +",284,0.9815 2025-09-22T14:00:50.936660,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49939,7 +49939,7 @@ class Problem { } }",161.04460000497056,,Completions generated successfully. Ready for display in your IDE.,6e3af699-4eb6-474d-8fd2-21abf2ffc9d0,1,deepseek-ai/deepseek-coder-1.3b-base," if (min < max) { -",134,1.0 +",134,1.0 2025-09-22T14:00:51.071570,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49986,7 +49986,7 @@ class Problem { }"," } }",133.9732999986154,,Completions generated successfully. Ready for display in your IDE.,592c39a2-c28e-4e81-95d4-f9d600a86614,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",104,1.0 +",104,1.0 2025-09-22T14:00:51.167687,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50006,7 +50006,7 @@ class Problem { } }",95.83159999601776,,Completions generated successfully. Ready for display in your IDE.,3ef02a04-c634-410c-9856-e33611c94feb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T14:00:51.293705,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -50042,7 +50042,7 @@ class Problem { ",126.43329999991693,,Completions generated successfully. Ready for display in your IDE.,fc527aa8-d1e3-44e3-b7ed-69636909ba8f,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",100,1.0 +",100,1.0 2025-09-22T14:00:51.438905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50057,7 +50057,7 @@ class Problem { public static float triangleArea(long a, long h) { "," } }",144.95309999620076,,Completions generated successfully. Ready for display in your IDE.,116e7d95-974d-46af-802d-3177a01fcdd0,1,deepseek-ai/deepseek-coder-1.3b-base," return ((a * h)/2); -",123,0.9755 +",123,0.9755 2025-09-22T14:00:52.134294,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -50100,7 +50100,7 @@ function max_fill(grid, capacity){ const c = grid[0].length; if (r === 0 || !c ){return ""Invalid Input""} - function",670,0.8946 + function",670,0.8946 2025-09-22T14:00:52.189290,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50132,7 +50132,7 @@ class Problem { return ret; } }",54.82740000297781,,Completions generated successfully. Ready for display in your IDE.,4e6e9c6a-1adc-4c90-b76c-27ce0acb25de,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T14:00:52.249663,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -50175,7 +50175,7 @@ function bf(planet1, planet2){ } return res; }",60.55510000442155,,Completions generated successfully. Ready for display in your IDE.,331ef0e8-b0d6-49be-bc5c-92661b4a75a6,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T14:00:52.527812,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -50197,7 +50197,7 @@ function find_closest_elements(numbers){ } ",277.73570000135805,,Completions generated successfully. Ready for display in your IDE.,b7ba423a-dfbf-4855-beec-dec4db7a1445,1,deepseek-ai/deepseek-coder-1.3b-base," if ((Math.abs(numbers[i] - numbers[j])) <= closest ){ -",255,0.9968 +",255,0.9968 2025-09-22T14:00:52.675683,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50221,7 +50221,7 @@ class Problem { } }",147.84769999823766,,Completions generated successfully. Ready for display in your IDE.,fc624532-fc3b-4329-a2b8-d04b80fb4987,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(delimeter); -",124,1.0 +",124,1.0 2025-09-22T14:00:52.903158,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -50246,7 +50246,7 @@ function make_palindrome(string){ } ",226.88170000037644,,Completions generated successfully. Ready for display in your IDE.,45359e0e-81f0-4925-8d4f-58841ff76cc8,1,deepseek-ai/deepseek-coder-1.3b-base," if (is_palindrome(string+=""_"")){ -",203,0.966 +",203,0.966 2025-09-22T14:00:52.999811,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -50275,7 +50275,7 @@ function split_words(txt){ } }",96.77969999756897,,Completions generated successfully. Ready for display in your IDE.,fb049a75-903b-4910-bf78-dd70dc6641d7,1,deepseek-ai/deepseek-coder-1.3b-base," else{ -",72,1.0 +",72,1.0 2025-09-22T14:00:53.703251,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50311,7 +50311,7 @@ class Problem { switch (digit) { case 1:c='1';break; case 2:c='2';break; - case 3:c='3",670,0.99 + case 3:c='3",670,0.99 2025-09-22T14:00:53.788245,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50343,7 +50343,7 @@ class Problem { } return out; }",,84.6873000045889,,Completions generated successfully. Ready for display in your IDE.,a67edd49-fb7c-45bb-a92f-95483d28e303,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:00:53.872081,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50386,7 +50386,7 @@ class Problem { } return false;",},83.63860000099521,,Completions generated successfully. Ready for display in your IDE.,9e7e5115-08d2-48a3-8352-33849fcbb5ca,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:53.925124,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -50418,7 +50418,7 @@ function prod_signs(arr){ } } ",53.12989999947604,,Completions generated successfully. Ready for display in your IDE.,62ddc260-4555-4867-a192-8e5355295263,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T14:00:54.032171,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -50437,7 +50437,7 @@ function triangle_area(a, b, c){ } ",106.77830000349786,,Completions generated successfully. Ready for display in your IDE.,2110ba2a-853a-4528-98e0-19e6bce5580b,1,deepseek-ai/deepseek-coder-1.3b-base," }else{ -",82,0.9892 +",82,0.9892 2025-09-22T14:00:54.280730,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -50471,7 +50471,7 @@ function count_nums(arr){ return count; }",248.0954000056954,,Completions generated successfully. Ready for display in your IDE.,6401d250-9aa8-41d5-879f-0044b3b195d6,1,deepseek-ai/deepseek-coder-1.3b-base," for(var j = 2; j < str.length; j++) { -",225,1.0 +",225,1.0 2025-09-22T14:00:54.519808,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50519,7 +50519,7 @@ class Problem { } }",238.67219999374356,,Completions generated successfully. Ready for display in your IDE.,2854e2e3-a5af-48b1-8e57-597d9e94998a,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList l_e = new ArrayList(); -",212,1.0 +",212,1.0 2025-09-22T14:00:54.613908,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -50538,7 +50538,7 @@ class Problem { ",93.42389999801526,,Completions generated successfully. Ready for display in your IDE.,080bc018-5c5b-4171-a701-9daabf3c1d6a,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T14:00:54.768855,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -50564,7 +50564,7 @@ function is_multiply_prime(a){ } }",155.10989999893354,,Completions generated successfully. Ready for display in your IDE.,478a40ff-aa0c-45af-9dc5-a840ebd8875a,1,deepseek-ai/deepseek-coder-1.3b-base," while(!isNaN(a)){ -",132,0.9257 +",132,0.9257 2025-09-22T14:00:54.868007,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50596,7 +50596,7 @@ class Problem { } }",99.0382999952999,,Completions generated successfully. Ready for display in your IDE.,4081eb53-a891-4300-b398-5ec6b3dd66ed,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:54.982429,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -50673,7 +50673,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",113.73189999721944,,Completions generated successfully. Ready for display in your IDE.,3e8bb3a6-1f67-4a3c-a905-8b8757d7ded4,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",87,1.0 +",87,1.0 2025-09-22T14:00:55.277620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50710,7 +50710,7 @@ class Problem { } }",295.0032000007923,,Completions generated successfully. Ready for display in your IDE.,e8817848-d9b7-43cd-91a2-6860684bb05c,1,deepseek-ai/deepseek-coder-1.3b-base," char c = paren_string.charAt(i); -",204,1.0 +",204,1.0 2025-09-22T14:00:55.634035,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50752,7 +50752,7 @@ class Problem { } }",356.8834000034258,,Completions generated successfully. Ready for display in your IDE.,08c00009-88df-4fcb-8ea8-a622e9f1f448,1,deepseek-ai/deepseek-coder-1.3b-base," int sm = (int)s.chars().filter(c -> Character.isLowerCase(c)).count(); -",329,1.0 +",329,1.0 2025-09-22T14:00:55.844169,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50790,7 +50790,7 @@ class Problem { } }",209.8755999977584,,Completions generated successfully. Ready for display in your IDE.,c0484343-991f-4989-b212-345ce0ba1508,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (chars[i] == '.') { -",191,0.9505 +",191,0.9505 2025-09-22T14:00:55.943894,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50810,7 +50810,7 @@ class Problem { }"," } }",98.99330000189366,,Completions generated successfully. Ready for display in your IDE.,0ed3158c-2778-498e-ae0d-d84c77a7c20c,1,deepseek-ai/deepseek-coder-1.3b-base," return results; -",81,1.0 +",81,1.0 2025-09-22T14:00:56.187236,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50875,7 +50875,7 @@ class Problem { } }",242.9688000047463,,Completions generated successfully. Ready for display in your IDE.,1aae2a67-0274-4b00-b33c-77dd4cba4c2d,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 2.3f) { -",217,1.0 +",217,1.0 2025-09-22T14:00:56.585954,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50925,7 +50925,7 @@ class Problem { }",399.07900000252994,,Completions generated successfully. Ready for display in your IDE.,e98d927a-1b40-4e7a-935c-2bea03d84c09,1,deepseek-ai/deepseek-coder-1.3b-base," if (year < 1900 && year > 2020) return valid_date; if (month == 2) { -",379,0.9664 +",379,0.9664 2025-09-22T14:00:57.173273,[1],"//Create a function that returns true if the last character // of a given string is an alphabetical character and is not // a part of a word, and false otherwise. @@ -50946,7 +50946,7 @@ function check_if_last_char_is_a_letter(txt){ }",586.2827999953879,,Completions generated successfully. Ready for display in your IDE.,aaa8ca71-59e7-40a9-8779-c408e4e80782,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0 ;i 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -50963,7 +50963,7 @@ function largest_prime_factor(n){ } }",},95.47249999741325,,Completions generated successfully. Ready for display in your IDE.,c7f3fb02-728d-4d64-857c-9725e97fe99f,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",72,0.9947 +",72,0.9947 2025-09-22T14:00:57.373806,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -50992,7 +50992,7 @@ function is_equal_to_sum_even(n){ else{ return false; }",,105.00670000328682,,Completions generated successfully. Ready for display in your IDE.,19668dec-35ea-46f6-9e68-819f38aeff72,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T14:00:57.439557,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -51007,7 +51007,7 @@ function count_distinct_characters(string){ } } return result.length;",,65.53870000061579,,Completions generated successfully. Ready for display in your IDE.,069f1032-2a72-4ff8-9a03-7a0c9750b965,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:57.616861,[1],"from typing import List def can_arrange(arr: List[int]) -> int: @@ -51027,7 +51027,7 @@ def can_arrange(arr: List[int]) -> int: return ans ",177.1387999979197,,Completions generated successfully. Ready for display in your IDE.,9c8f917a-5d0c-407d-9ede-3956034fcdc6,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(arr)): -",154,1.0 +",154,1.0 2025-09-22T14:00:57.735679,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -51053,7 +51053,7 @@ def find_max(words: List[str]) -> str: elif unique_num == max_num: if word < max_word: max_word = word",,118.44360000395682,,Completions generated successfully. Ready for display in your IDE.,5dfb6088-e1b4-48ca-9cb5-e94375adfe44,1,deepseek-ai/deepseek-coder-1.3b-base," - return max_word",93,0.9576 + return max_word",93,0.9576 2025-09-22T14:00:57.802146,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51070,7 +51070,7 @@ class Problem { public static String concatenate(ArrayList strings) { return strings.stream().collect(Collectors.joining()); }",,65.62539999868022,,Completions generated successfully. Ready for display in your IDE.,20f0f85b-d818-4242-a134-c370a3a6689a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,0.7409 +}",42,0.7409 2025-09-22T14:00:57.961012,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -51110,7 +51110,7 @@ function match_parens(lst){ return 'No'; }",158.69129999919096,,Completions generated successfully. Ready for display in your IDE.,032a3ebf-54e6-4904-be40-4341f513c2fb,1,deepseek-ai/deepseek-coder-1.3b-base," for (const p of s){ -",130,1.0 +",130,1.0 2025-09-22T14:00:58.055833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51143,7 +51143,7 @@ class Problem { return new String(arr); } }",95.07959999609739,,Completions generated successfully. Ready for display in your IDE.,9f958e57-4823-41b3-a956-0ab2733d7f2f,1,deepseek-ai/deepseek-coder-1.3b-base," - ",43,1.0 + ",43,1.0 2025-09-22T14:00:58.152878,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -51177,7 +51177,7 @@ class Problem { prime_words.append(word) return ' '.join(prime_words)",96.71569999773055,,Completions generated successfully. Ready for display in your IDE.,9efe3a3f-a8dd-4611-b97f-31228aaab91a,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T14:00:58.227923,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -51204,7 +51204,7 @@ function monotonic(l){ return true; } ",74.68030000018189,,Completions generated successfully. Ready for display in your IDE.,36ea359f-d2b8-4665-9063-fe7f3c44ead4,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",52,0.981 +",52,0.981 2025-09-22T14:00:58.373599,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -51248,7 +51248,7 @@ function histogram(test){ } ",145.3814999986207,,Completions generated successfully. Ready for display in your IDE.,6656e8fb-198a-43da-882f-25d2a0d01ef3,1,deepseek-ai/deepseek-coder-1.3b-base," max = count[key]; -",120,1.0 +",120,1.0 2025-09-22T14:00:58.491001,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -51269,7 +51269,7 @@ function even_odd_count(num){ } ",117.6019999984419,,Completions generated successfully. Ready for display in your IDE.,67c96793-17a8-41e2-bac4-a5d408a43297,1,deepseek-ai/deepseek-coder-1.3b-base," var odds = 0; -",93,1.0 +",93,1.0 2025-09-22T14:00:58.573893,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51290,7 +51290,7 @@ class Problem { public static boolean rightAngleTriangle(long a, long b, long c) { return a*a + b*b == c*c || b*b + c*c == a*a || c*c + a*a == b*b;",},82.65559999563266,,Completions generated successfully. Ready for display in your IDE.,36e1cb35-93be-4606-98e5-65a2c8ecd8bf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:58.660665,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -51310,7 +51310,7 @@ function unique_digits(x){ ",86.22900000045775,,Completions generated successfully. Ready for display in your IDE.,d6cc2228-a89e-41e5-8b61-2bea146db424,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:58.783722,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51349,7 +51349,7 @@ class Problem { } }",123.06889999308623,,Completions generated successfully. Ready for display in your IDE.,90342feb-ff48-4fab-b68b-d43ccaf709c2,1,deepseek-ai/deepseek-coder-1.3b-base," index = 0; -",98,0.9481 +",98,0.9481 2025-09-22T14:00:59.008011,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51393,7 +51393,7 @@ class Problem { } }",224.03249999479158,,Completions generated successfully. Ready for display in your IDE.,d24d8f1a-c511-4dbb-b4ba-bcda57d83e1f,1,deepseek-ai/deepseek-coder-1.3b-base," else if (op.get(i).equals(""**"")) -",196,1.0 +",196,1.0 2025-09-22T14:00:59.182910,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -51416,7 +51416,7 @@ function factorize(n){ } ",174.67889999534236,,Completions generated successfully. Ready for display in your IDE.,f4dba343-88bb-4545-97ca-0dcf7342d227,1,deepseek-ai/deepseek-coder-1.3b-base," if ((n % factor === 0)) { -",147,0.9844 +",147,0.9844 2025-09-22T14:00:59.292516,[1],"from typing import List def select_words(s: str, n: int) -> List[str]: @@ -51448,7 +51448,7 @@ def select_words(s: str, n: int) -> List[str]: return result ",108.91020000417484,,Completions generated successfully. Ready for display in your IDE.,ca524985-3943-4885-b24d-5179fb826b8a,1,deepseek-ai/deepseek-coder-1.3b-base," result = [] -",83,1.0 +",83,1.0 2025-09-22T14:00:59.574479,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51490,7 +51490,7 @@ class Problem { } }",281.52959999715677,,Completions generated successfully. Ready for display in your IDE.,9b4f3dea-4e24-4f06-b545-a21783d45a19,1,deepseek-ai/deepseek-coder-1.3b-base," if (s3.charAt(i) == '(') c1 += 1; -",255,1.0 +",255,1.0 2025-09-22T14:00:59.680152,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -51512,7 +51512,7 @@ def odd_count(lst: List[str]) -> List[str]: return output ",106.06429999461398,,Completions generated successfully. Ready for display in your IDE.,a7fe716d-868f-4141-8105-4d483a104011,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:59.814479,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -51542,7 +51542,7 @@ def count_nums(arr: List[int]) -> int: return n ",133.97819999954663,,Completions generated successfully. Ready for display in your IDE.,03ccb214-e37c-46e8-b562-2140a5e45dd1,1,deepseek-ai/deepseek-coder-1.3b-base," s += int(j) -",112,1.0 +",112,1.0 2025-09-22T14:00:59.979910,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -51586,7 +51586,7 @@ function histogram(test){ } ",165.2570000005653,,Completions generated successfully. Ready for display in your IDE.,053a9368-0f61-4f8b-b337-5ac0dd17534a,1,deepseek-ai/deepseek-coder-1.3b-base," //console.log(lst) -",141,0.8655 +",141,0.8655 2025-09-22T14:01:00.132407,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51664,7 +51664,7 @@ class Problem { } }",152.25910000299336,,Completions generated successfully. Ready for display in your IDE.,4c123407-e258-40cc-b0f6-51aa269202a2,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 1; -",112,0.9202 +",112,0.9202 2025-09-22T14:01:00.299853,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51702,7 +51702,7 @@ class Problem { } }",166.62360000191256,,Completions generated successfully. Ready for display in your IDE.,1dc27089-f598-4b0b-bd7d-a2a2dbe253a4,1,deepseek-ai/deepseek-coder-1.3b-base," sum1 += s.length(); -",138,1.0 +",138,1.0 2025-09-22T14:01:00.475545,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -51732,7 +51732,7 @@ function f(n){ return result; }",175.96559999947203,,Completions generated successfully. Ready for display in your IDE.,a43e92f5-8f9e-4c70-8e54-92d95a4f954a,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here var result = []; -",153,0.9316 +",153,0.9316 2025-09-22T14:01:00.616898,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51777,7 +51777,7 @@ class Problem { } }",140.93850000062957,,Completions generated successfully. Ready for display in your IDE.,5f6d2ba9-038f-4543-9560-4f775a16f5e4,1,deepseek-ai/deepseek-coder-1.3b-base," boolean isPrime; -",110,0.9818 +",110,0.9818 2025-09-22T14:01:00.814384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51819,7 +51819,7 @@ class Problem { } }",197.55860000441317,,Completions generated successfully. Ready for display in your IDE.,08f30600-fe16-4c3b-9b47-3ec62b390d2b,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append(s.charAt(i)); -",173,1.0 +",173,1.0 2025-09-22T14:01:01.224546,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51844,7 +51844,7 @@ class Problem { } }",409.49080000427784,,Completions generated successfully. Ready for display in your IDE.,14b1b989-eb94-4c6f-82fd-84cd882af18c,1,deepseek-ai/deepseek-coder-1.3b-base," ret = ((double)(l.get(size / 2 - 1) + l.get(size / 2))) / 2d; -",385,0.9886 +",385,0.9886 2025-09-22T14:01:01.484092,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -51921,7 +51921,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",259.72919999912847,,Completions generated successfully. Ready for display in your IDE.,e4a1d9f0-e78d-4f6c-a09f-c3663a9c49e4,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, str) and isinstance(b, float): -",232,0.975 +",232,0.975 2025-09-22T14:01:01.887819,[1],"from typing import List def sorted_list_sum(lst: List[str]) -> List[str]: @@ -51943,7 +51943,7 @@ def sorted_list_sum(lst: List[str]) -> List[str]: """""" ",,403.7968999982695,,Completions generated successfully. Ready for display in your IDE.,500ce5ae-5747-4150-997f-ce7d4c9ee77e,1,deepseek-ai/deepseek-coder-1.3b-base," lst = [i for i in lst if len(i) % 2 == 0] lst.sort() - return lst",380,0.9563 + return lst",380,0.9563 2025-09-22T14:01:01.932456,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -51970,7 +51970,7 @@ function solve(s){ } return s.split('').reverse().join(''); } -",44.17959999409504,,Completions generated successfully. Ready for display in your IDE.,999908d5-f8ab-45d0-bce7-393ad422b665,1,deepseek-ai/deepseek-coder-1.3b-base,,22,0.859 +",44.17959999409504,,Completions generated successfully. Ready for display in your IDE.,999908d5-f8ab-45d0-bce7-393ad422b665,1,deepseek-ai/deepseek-coder-1.3b-base,,22,0.859 2025-09-22T14:01:02.102067,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -51996,7 +51996,7 @@ function separate_paren_groups(paren_string){ } ",169.48609999963082,,Completions generated successfully. Ready for display in your IDE.,1bed5597-7fd8-44bf-8925-1a05bfdeb4cc,1,deepseek-ai/deepseek-coder-1.3b-base," group += paren_string[i]; -",145,1.0 +",145,1.0 2025-09-22T14:01:02.216174,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52029,7 +52029,7 @@ class Problem { }"," } }",113.44200000166893,,Completions generated successfully. Ready for display in your IDE.,f3650218-b093-4377-be16-0aa1cb6c6943,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",89,1.0 +",89,1.0 2025-09-22T14:01:02.477528,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52073,7 +52073,7 @@ class Problem { } }",261.61470000079134,,Completions generated successfully. Ready for display in your IDE.,d987235a-d757-4724-918b-033b86ea630c,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < word.length(); i++) { -",235,1.0 +",235,1.0 2025-09-22T14:01:03.177415,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52120,7 +52120,7 @@ class Problem { while (!Character.isLetter(arr[j])){ j--; } - Collections.reverse(",674,0.974 + Collections.reverse(",674,0.974 2025-09-22T14:01:03.297905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52161,7 +52161,7 @@ class Problem { }"," } }",120.28180000197608,,Completions generated successfully. Ready for display in your IDE.,6808e0fb-1a1d-4daf-8d12-a702178f5b0f,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",92,1.0 +",92,1.0 2025-09-22T14:01:03.409886,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -52229,7 +52229,7 @@ function do_algebra(operator, operand){ } ",111.61980000179028,,Completions generated successfully. Ready for display in your IDE.,21d7fbbe-185c-483a-8d58-273c03bc0b2d,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",82,1.0 +",82,1.0 2025-09-22T14:01:03.500425,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -52258,7 +52258,7 @@ function will_it_fly(q, w){ return balanced && sum <= w; }",90.69269999599783,,Completions generated successfully. Ready for display in your IDE.,fcb6c466-d519-4409-8f20-fab8fd06b136,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T14:01:03.725009,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -52325,7 +52325,7 @@ function numerical_letter_grade(grades){ return grades_array; }",223.75550000288058,,Completions generated successfully. Ready for display in your IDE.,1dc94bd3-360f-458e-87bb-1eb14ae416a0,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 0.7){ -",191,1.0 +",191,1.0 2025-09-22T14:01:03.906841,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52365,7 +52365,7 @@ class Problem { } }",182.10790000011912,,Completions generated successfully. Ready for display in your IDE.,36e4da56-47e7-4341-b851-01c4c4702212,1,deepseek-ai/deepseek-coder-1.3b-base," long z = x.get(i); -",149,1.0 +",149,1.0 2025-09-22T14:01:04.014853,[1],"//From an array of integers, remove all elements that occur more than once. // Keep order of elements left the same as in the input. // >>> remove_duplicates([1, 2, 3, 2, 4]) @@ -52379,7 +52379,7 @@ function remove_duplicates(numbers){ });","} ",106.98660000343807,,Completions generated successfully. Ready for display in your IDE.,aca522a1-c2d5-4cbb-9ff4-bbe60b59efb6,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",83,0.9937 +",83,0.9937 2025-09-22T14:01:04.176910,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52425,7 +52425,7 @@ class Problem { }",162.44410000217613,,Completions generated successfully. Ready for display in your IDE.,dbe8db40-bb51-43b4-978c-b883bc1d1779,1,deepseek-ai/deepseek-coder-1.3b-base," long sum = 0; -",133,0.9065 +",133,0.9065 2025-09-22T14:01:04.427333,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52460,7 +52460,7 @@ class Problem { } }",249.66870000207564,,Completions generated successfully. Ready for display in your IDE.,543220eb-4459-44e9-b82e-d2609d322cd1,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sz; i++) { -",222,1.0 +",222,1.0 2025-09-22T14:01:04.712905,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -52493,7 +52493,7 @@ function correct_bracketing(brackets){ }",285.683499998413,,Completions generated successfully. Ready for display in your IDE.,89cfc957-0cd8-409f-bd07-1c58b93bbe3e,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < brackets.length; i++ ){ -",260,0.9907 +",260,0.9907 2025-09-22T14:01:04.790160,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -52521,7 +52521,7 @@ function is_happy(s){ } } return happy;",,77.19179999548942,,Completions generated successfully. Ready for display in your IDE.,3ce67ca6-b5fa-4de2-9022-e8ac6de894b0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,0.9689 +}",49,0.9689 2025-09-22T14:01:04.877622,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52553,7 +52553,7 @@ class Problem { } return isPalindrome;",},87.24240000447026,,Completions generated successfully. Ready for display in your IDE.,0062b07f-623b-497a-aad1-536c1b15d712,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:01:05.123002,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52593,7 +52593,7 @@ class Problem { long left = Math.max(0, remaining - need);"," } }",245.2248999979929,,Completions generated successfully. Ready for display in your IDE.,a4e0a70d-6881-4295-8bc3-38e5b7ff3dca,1,deepseek-ai/deepseek-coder-1.3b-base," return new ArrayList<>(Arrays.asList(total, left)); -",214,1.0 +",214,1.0 2025-09-22T14:01:05.334148,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -52632,7 +52632,7 @@ function valid_date(date){ } ",210.48380000138422,,Completions generated successfully. Ready for display in your IDE.,86f3ecca-b862-4eae-b5b7-dabba5d1041e,1,deepseek-ai/deepseek-coder-1.3b-base," var month = parseInt(date_arr[0]); -",183,1.0 +",183,1.0 2025-09-22T14:01:05.443660,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52685,7 +52685,7 @@ class Problem { newArr.add((long)minIndex); return newArr;",},109.80600000038976,,Completions generated successfully. Ready for display in your IDE.,0a6b10df-75ca-4d2d-97b3-b404fc443c87,1,deepseek-ai/deepseek-coder-1.3b-base," } -",80,1.0 +",80,1.0 2025-09-22T14:01:06.136464,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -52707,7 +52707,7 @@ class Problem { ",692.3715000011725,,Completions generated successfully. Ready for display in your IDE.,dfd34d8a-295a-4df9-ba4b-6857c2949a89,1,deepseek-ai/deepseek-coder-1.3b-base," if ord(i) >= 65 and ord(i) <= 90: #if uppercase output_string += chr((ord(i)-23-1)) - elif ord(i) >= 97 and ord(i) <= 122: #if lower",668,0.9623 + elif ord(i) >= 97 and ord(i) <= 122: #if lower",668,0.9623 2025-09-22T14:01:06.243469,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -52733,7 +52733,7 @@ def unique_digits(x: List[int]) -> List[int]: return sorted(x) ",107.02060000039637,,Completions generated successfully. Ready for display in your IDE.,999cf529-42eb-4268-a1fb-5d0075dfa397,1,deepseek-ai/deepseek-coder-1.3b-base," temp=i -",84,1.0 +",84,1.0 2025-09-22T14:01:06.307483,[1],"//Given a positive integer n, you have to make a pile of n levels of stones. // The first level has n stones. // The number of stones in the next level is: @@ -52750,7 +52750,7 @@ function make_a_pile(n){ arr.push(n + (i * 2)); } return arr;",,63.82099999609636,,Completions generated successfully. Ready for display in your IDE.,264eff55-8d61-4317-b70f-dcfc89b98594,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:01:06.381880,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52785,7 +52785,7 @@ class Problem { return (lst1.stream().allMatch(x -> x % 2 == 0)) ? ""YES"" : ""NO""; } }",74.14750000316417,,Completions generated successfully. Ready for display in your IDE.,3456bb7c-8277-4922-9fac-7dcfe77679bf,1,deepseek-ai/deepseek-coder-1.3b-base," - ",49,0.8545 + ",49,0.8545 2025-09-22T14:01:06.480167,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -52807,7 +52807,7 @@ def odd_count(lst: List[str]) -> List[str]: output.append(f""the number of odd elements {count}n the str{count}ng {count} of the {count}nput."") return output ",97.84819999913452,,Completions generated successfully. Ready for display in your IDE.,d6e05da7-ce47-40a0-ad89-d9e30085fa1a,1,deepseek-ai/deepseek-coder-1.3b-base," output = [] -",73,1.0 +",73,1.0 2025-09-22T14:01:06.566532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52831,7 +52831,7 @@ class Problem { } }",86.44570000615204,,Completions generated successfully. Ready for display in your IDE.,058eb06b-20a7-43d3-82a9-d036e6ce8a7e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:01:06.691846,[1],"from typing import Tuple def intersection(interval1: Tuple[int, int], interval2: Tuple[int, int]) -> str: @@ -52862,7 +52862,7 @@ def intersection(interval1: Tuple[int, int], interval2: Tuple[int, int]) -> str: if start > end:"," return ""YES"" if end - start == 2 else ""NO"" ",125.12270000297576,,Completions generated successfully. Ready for display in your IDE.,e383f497-4dce-4ec7-b8a8-68de2da115b8,1,deepseek-ai/deepseek-coder-1.3b-base," return ""NO"" -",99,1.0 +",99,1.0 2025-09-22T14:01:06.906968,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52927,7 +52927,7 @@ class Problem { } }",215.17989999847487,,Completions generated successfully. Ready for display in your IDE.,3b8c0836-7b6e-4ffe-bd39-d365c0ba2c62,1,deepseek-ai/deepseek-coder-1.3b-base," float currentGrade = grades.get(i); -",186,1.0 +",186,1.0 2025-09-22T14:01:07.479625,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52944,7 +52944,7 @@ class Problem { public static ArrayList filterIntegers(ArrayList values) { "," } }",572.0447999992757,,Completions generated successfully. Ready for display in your IDE.,ed3e5779-1ddf-447b-8016-51d031bbda24,1,deepseek-ai/deepseek-coder-1.3b-base," return (ArrayList)(values).stream().filter(value -> value instanceof Long || value instanceof Integer).map(value -> ((Number)value).longValue()).collect(Collectors.toCollection(ArrayList::new)); -",546,0.9899 +",546,0.9899 2025-09-22T14:01:07.581743,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -52971,7 +52971,7 @@ function next_smallest(lst){ return second_smallest; }",101.89799999352545,,Completions generated successfully. Ready for display in your IDE.,9a9098b6-5192-4950-9f65-6a6400449c1d,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",81,1.0 +",81,1.0 2025-09-22T14:01:07.656664,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -52986,7 +52986,7 @@ function filter_integers(values){ } ",74.33250000030966,,Completions generated successfully. Ready for display in your IDE.,59504b74-87e5-4a34-a23f-dd82b5933071,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",53,1.0 +",53,1.0 2025-09-22T14:01:07.846722,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -53009,7 +53009,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: return result ",190.28939999407157,,Completions generated successfully. Ready for display in your IDE.,6fe671be-e877-4f72-a542-57a0f9482e3c,1,deepseek-ai/deepseek-coder-1.3b-base," for parens in paren_string.split(): -",167,0.9899 +",167,0.9899 2025-09-22T14:01:07.940209,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -53027,7 +53027,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: if text[i] != text[-i - 1]: return False"," ",93.044799999916,,Completions generated successfully. Ready for display in your IDE.,699ac823-32b6-45ac-944b-3f198f15e19f,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",62,1.0 + return True",62,1.0 2025-09-22T14:01:08.034848,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53067,7 +53067,7 @@ class Problem { } }",94.07780000037747,,Completions generated successfully. Ready for display in your IDE.,036d2584-4cb6-4cd0-9038-6da7ca23dc89,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:01:08.101050,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53107,7 +53107,7 @@ class Problem { return res; } }",66.00529999559512,,Completions generated successfully. Ready for display in your IDE.,3a9d66ee-aca1-414d-8140-ecafbf26c42d,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:01:08.670176,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -53125,7 +53125,7 @@ function count_distinct_characters(string){ console.log('The number of unique letters in the given word is: ' + arr.length); //loop through array and check to see if letter already exists within our new empty object/array called ""result"" -",543,0.8458 +",543,0.8458 2025-09-22T14:01:08.737618,[1],"//You're a hungry rabbit, and you already have eaten a certain number of carrots, // but now you need to eat more carrots to complete the day's meals. // you should return an array of [ total number of eaten carrots after your meals, @@ -53155,7 +53155,7 @@ function count_distinct_characters(string){ function eat(number, need, remaining){ if (need > remaining) return [number + remaining, 0]; else return [number + need, remaining - need];",,66.6553000046406,,Completions generated successfully. Ready for display in your IDE.,6e5e41ce-d265-4e37-8a13-62b5e380a480,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T14:01:08.826528,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53185,7 +53185,7 @@ class Problem { factors.add(n); return factors;",},88.88699999806704,,Completions generated successfully. Ready for display in your IDE.,6782bbfb-37fd-4a6e-a801-7598b4469bab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:01:09.047889,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53220,7 +53220,7 @@ class Problem { } }",220.8258999962709,,Completions generated successfully. Ready for display in your IDE.,7e00b050-88c3-4c86-bc7c-5206de912b84,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (i % 4 != 0) { -",196,0.9856 +",196,0.9856 2025-09-22T14:01:09.141745,[1],"def strlen(string: str) -> int: """""" Return length of given string >>> strlen('') @@ -53233,7 +53233,7 @@ class Problem { return length ",94.18649999861373,,Completions generated successfully. Ready for display in your IDE.,87a0c986-b99b-4aae-81f4-edd13fed96dd,1,deepseek-ai/deepseek-coder-1.3b-base," length = 0 -",72,1.0 +",72,1.0 2025-09-22T14:01:09.229360,[1],"//Task // Write a function that takes a string as input and returns the sum of the upper characters only' // ASCII codes. @@ -53258,7 +53258,7 @@ function digitSum(s){ return sum; }",87.56410000205506,,Completions generated successfully. Ready for display in your IDE.,e4e03568-56ea-4287-af2a-320c785305ef,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T14:01:09.434721,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53297,7 +53297,7 @@ class Problem { } }",205.16079999651993,,Completions generated successfully. Ready for display in your IDE.,b9f714de-6daa-4f59-9269-abe83535700c,1,deepseek-ai/deepseek-coder-1.3b-base," long currSum = nums.get(i); -",181,1.0 +",181,1.0 2025-09-22T14:01:09.544566,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53337,7 +53337,7 @@ class Problem { } }",109.22539999592118,,Completions generated successfully. Ready for display in your IDE.,e2f4641e-2987-4328-b26a-f30ff73cdc58,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",84,1.0 +",84,1.0 2025-09-22T14:01:09.879978,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53354,7 +53354,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { "," } }",334.9744000006467,,Completions generated successfully. Ready for display in your IDE.,d22540f4-c490-4a30-bdb9-711e5f24ecb0,1,deepseek-ai/deepseek-coder-1.3b-base," return l.stream().filter(i -> i > 0).collect(Collectors.toCollection(ArrayList::new)); -",311,0.9679 +",311,0.9679 2025-09-22T14:01:09.974916,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -53393,7 +53393,7 @@ function valid_date(date){ } ",95.6240000014077,,Completions generated successfully. Ready for display in your IDE.,c986257e-2000-4360-a4c2-58d87dbcbed6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:01:10.061243,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53429,7 +53429,7 @@ class Problem { } return max; }",,85.43840000493219,,Completions generated successfully. Ready for display in your IDE.,58392109-a47f-44e3-a180-30c2d88dda1c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T14:01:10.180758,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53459,7 +53459,7 @@ class Problem { return res; } }",119.02420000114944,,Completions generated successfully. Ready for display in your IDE.,ce3f3acc-907a-48b7-a786-bcfab55ab8bc,1,deepseek-ai/deepseek-coder-1.3b-base," int res = 0; -",95,1.0 +",95,1.0 2025-09-22T14:01:10.358228,[1],"from typing import List def count_up_to(n: int) -> List[int]: @@ -53491,7 +53491,7 @@ def count_up_to(n: int) -> List[int]: return result ",177.94470000080764,,Completions generated successfully. Ready for display in your IDE.,7bf88ae6-d7da-4b89-9702-85c5999eefe4,1,deepseek-ai/deepseek-coder-1.3b-base," for divisor in range(2, number): -",154,0.9497 +",154,0.9497 2025-09-22T14:01:10.465688,[1],"from typing import List def factorize(n: int) -> List[int]: @@ -53513,7 +53513,7 @@ def factorize(n: int) -> List[int]: return factors ",107.2379999968689,,Completions generated successfully. Ready for display in your IDE.,165d5b13-9109-465d-a289-1ea96c4a9ba1,1,deepseek-ai/deepseek-coder-1.3b-base," i = 2 -",83,1.0 +",83,1.0 2025-09-22T14:01:10.723745,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -53531,7 +53531,7 @@ function choose_num(x, y){ } ",257.0287000053213,,Completions generated successfully. Ready for display in your IDE.,4321cc9f-4e79-452d-945b-ffd9995af3f2,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=y-1; i>=x+1; --i ){ -",235,0.9691 +",235,0.9691 2025-09-22T14:01:10.818587,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -53556,7 +53556,7 @@ function choose_num(x, y){ ",94.68709999782732,,Completions generated successfully. Ready for display in your IDE.,bf1f519e-9490-4757-ad56-45a9842e0638,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",71,1.0 +",71,1.0 2025-09-22T14:01:10.916618,[1],"from typing import List def monotonic(l: List[int]) -> bool: @@ -53571,7 +53571,7 @@ def monotonic(l: List[int]) -> bool: if l == sorted(l) or l == sorted(l, reverse=True):"," return False ",98.06229999958305,,Completions generated successfully. Ready for display in your IDE.,6662215f-5e49-4775-a76d-94096a819f81,1,deepseek-ai/deepseek-coder-1.3b-base," return True -",73,1.0 +",73,1.0 2025-09-22T14:01:10.984543,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -53638,7 +53638,7 @@ function do_algebra(operator, operand){ return answer; } ",67.97489999735262,,Completions generated successfully. Ready for display in your IDE.,a973bafe-43c5-429b-af0b-ae35919d04b6,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:01:11.124291,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -53668,7 +53668,7 @@ def count_nums(arr: List[int]) -> int: return n ",139.78920000226935,,Completions generated successfully. Ready for display in your IDE.,379f8a7b-1cf8-4c32-bf3d-f23c00b308a0,1,deepseek-ai/deepseek-coder-1.3b-base," for i in arr: -",103,1.0 +",103,1.0 2025-09-22T14:01:11.240065,[1],"from typing import List def add_elements(arr: List[int], k: int) -> int: @@ -53690,7 +53690,7 @@ def add_elements(arr: List[int], k: int) -> int: if num < 100: total += num",,114.96530000295024,,Completions generated successfully. Ready for display in your IDE.,ea328161-1d37-4c3c-9be2-99ac882eebdd,1,deepseek-ai/deepseek-coder-1.3b-base," - return total",93,0.9648 + return total",93,0.9648 2025-09-22T14:01:11.447194,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -53727,7 +53727,7 @@ def add_elements(arr: List[int], k: int) -> int: return False return True",207.35349999449681,,Completions generated successfully. Ready for display in your IDE.,1fe384cc-add3-4ce4-b14d-e80ac6de50b0,1,deepseek-ai/deepseek-coder-1.3b-base," if (len(date_list) != 3): -",182,0.9541 +",182,0.9541 2025-09-22T14:01:11.584281,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -53770,7 +53770,7 @@ def pluck(arr: List[int]) -> List[int]: return [min(evens), arr.index(min(evens))] ",136.4508000042406,,Completions generated successfully. Ready for display in your IDE.,561b33eb-e3d6-4c86-823a-c88b28e1435d,1,deepseek-ai/deepseek-coder-1.3b-base," if not evens: -",111,1.0 +",111,1.0 2025-09-22T14:01:11.665231,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53837,7 +53837,7 @@ class Problem { } } }",81.22330000333022,,Completions generated successfully. Ready for display in your IDE.,0b21cf17-3782-4bb8-a13d-b9b56e486471,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T14:01:12.390484,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -53922,7 +53922,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return None if __name__ == ""__main__"": - import",694,0.9848 + import",694,0.9848 2025-09-22T14:01:12.586672,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53954,7 +53954,7 @@ class Problem { } }",196.6251000048942,,Completions generated successfully. Ready for display in your IDE.,4002a85c-2ac7-412c-a9ab-677cb063f2b4,1,deepseek-ai/deepseek-coder-1.3b-base," out.add(numbers.get(i)); -",172,1.0 +",172,1.0 2025-09-22T14:01:12.629980,[1],"from typing import List def remove_duplicates(numbers: List[int]) -> List[int]: @@ -53966,7 +53966,7 @@ def remove_duplicates(numbers: List[int]) -> List[int]: numbers.sort()"," return numbers ",42.70679999899585,,Completions generated successfully. Ready for display in your IDE.,c0e19921-e979-4183-ae9d-2aef1e666ff9,1,deepseek-ai/deepseek-coder-1.3b-base," -",23,0.6808 +",23,0.6808 2025-09-22T14:01:12.770161,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -54002,7 +54002,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",139.77670000167564,,Completions generated successfully. Ready for display in your IDE.,90625d3d-8585-45dd-b93a-4cad6d81a7a1,1,deepseek-ai/deepseek-coder-1.3b-base," if c == '(': -",116,1.0 +",116,1.0 2025-09-22T14:01:13.500698,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54072,7 +54072,7 @@ class Problem { private static void main(String[] args){ System.out.println(""\n\n"" + ""Problem 6: Grading Algorithm"" + ""\n\n""); System.out.print(""Input: ""); -",701,0.9441 +",701,0.9441 2025-09-22T14:01:13.615643,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -54121,7 +54121,7 @@ function check_dict_case(dict){ return true; }",115.10759999509901,,Completions generated successfully. Ready for display in your IDE.,5dbd9e2c-a239-4d1c-8dfc-3466f9b2cf5a,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",87,1.0 +",87,1.0 2025-09-22T14:01:13.701241,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -54143,7 +54143,7 @@ function fizz_buzz(n){ } ",84.83040000282926,,Completions generated successfully. Ready for display in your IDE.,e2eeaa3a-7e5d-4558-b431-59ac473d51e6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:01:13.795441,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54176,7 +54176,7 @@ class Problem { } }",94.64129999832949,,Completions generated successfully. Ready for display in your IDE.,64b8e2b1-5b4e-46ee-9d24-e7b8cce8425c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:13.942838,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -54204,7 +54204,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: else: return ""YES""",146.91349999338854,,Completions generated successfully. Ready for display in your IDE.,c9d134c1-d498-4590-a10c-9e1247e08d32,1,deepseek-ai/deepseek-coder-1.3b-base," for x in lst1: -",122,1.0 +",122,1.0 2025-09-22T14:01:14.642136,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -54225,7 +54225,7 @@ function longest(strings){ } ",699.1903000016464,,Completions generated successfully. Ready for display in your IDE.,5e700835-9975-4ee8-828c-e0333388f418,1,deepseek-ai/deepseek-coder-1.3b-base," else if ((strings[i] === '') && !isNaN((parseInt('' + strings)))){ //if string contains only numbers and no letters or symbols then it will be falsey so we can use this to check that as well: console.log(""string was a number""); -",676,0.8404 +",676,0.8404 2025-09-22T14:01:14.722959,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -54254,7 +54254,7 @@ function sort_third(l){ } return x; }",80.64509999530856,,Completions generated successfully. Ready for display in your IDE.,38c33ff2-9cd5-474e-b836-bc8201c7677c,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",58,1.0 +",58,1.0 2025-09-22T14:01:15.403517,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -54286,7 +54286,7 @@ function hex_key(num){ return result; }",680.4918999987422,,Completions generated successfully. Ready for display in your IDE.,f0d7783e-4341-44cd-bd70-0da9d70a69ca,1,deepseek-ai/deepseek-coder-1.3b-base," //array with all possible values in order from lowest value upto highest one const primes = [2, 3, 5, 7]; // array containing only those which are divisible by these three numbers -> 2, 3, 5 & 7 respectively -",655,0.888 +",655,0.888 2025-09-22T14:01:15.500315,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54322,7 +54322,7 @@ class Problem { } }",95.77540000464069,,Completions generated successfully. Ready for display in your IDE.,0459e577-0ce3-4a77-9e49-a40503d88657,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:15.747159,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54387,7 +54387,7 @@ class Problem { } }",247.23390000144718,,Completions generated successfully. Ready for display in your IDE.,61a7de69-7223-4fda-a709-b92802b32c99,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 3.3f) { -",216,1.0 +",216,1.0 2025-09-22T14:01:15.937279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54419,7 +54419,7 @@ class Problem { } }",189.53329999931157,,Completions generated successfully. Ready for display in your IDE.,01203831-649b-43af-8077-a813d2d5870a,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList out = new ArrayList<>(); -",164,0.9703 +",164,0.9703 2025-09-22T14:01:16.069768,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -54440,7 +54440,7 @@ function sort_array(arr){ }); }",132.41829999606125,,Completions generated successfully. Ready for display in your IDE.,65ed7c63-9721-41e5-8f0a-1cecce8bbf5b,1,deepseek-ai/deepseek-coder-1.3b-base," return a - b; -",103,1.0 +",103,1.0 2025-09-22T14:01:16.289054,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -54507,7 +54507,7 @@ function numerical_letter_grade(grades){ return grades_array; }",219.07659999851603,,Completions generated successfully. Ready for display in your IDE.,676b1312-a0e5-4540-8e0f-6a8ef8d3466b,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 0.0){ -",190,1.0 +",190,1.0 2025-09-22T14:01:16.643993,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -54532,7 +54532,7 @@ function encode(message){ // not a vowel so just add to string without changing casing or anything result += letter; } -",329,0.8939 +",329,0.8939 2025-09-22T14:01:16.696322,[1],"from typing import List def f(n: int) -> List[int]: @@ -54553,7 +54553,7 @@ def f(n: int) -> List[int]: else: a.append(sum(range(1,i+1))) return a - pass",52.21030000393512,,Completions generated successfully. Ready for display in your IDE.,b0100fec-ff88-4482-ba87-73a539e9bcbe,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 + pass",52.21030000393512,,Completions generated successfully. Ready for display in your IDE.,b0100fec-ff88-4482-ba87-73a539e9bcbe,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:01:16.805654,[1],"//From an array of integers, remove all elements that occur more than once. // Keep order of elements left the same as in the input. // >>> remove_duplicates([1, 2, 3, 2, 4]) @@ -54567,7 +54567,7 @@ function remove_duplicates(numbers){ return result; } ",108.93360000045504,,Completions generated successfully. Ready for display in your IDE.,63112131-247b-4258-8080-faa3f40d3bf8,1,deepseek-ai/deepseek-coder-1.3b-base," var result = []; -",83,1.0 +",83,1.0 2025-09-22T14:01:16.871014,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -54599,7 +54599,7 @@ function prod_signs(arr){ } } ",65.20050000108313,,Completions generated successfully. Ready for display in your IDE.,c7ff4e04-026e-4da7-ae7a-1b1dab544948,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,1.0 + ",42,1.0 2025-09-22T14:01:16.990013,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -54619,7 +54619,7 @@ function prod_signs(arr){ output_string = """" for i in s: output_string += input_string[(input_string.index(i)+4)%26]",,118.78470000374364,,Completions generated successfully. Ready for display in your IDE.,622872b6-6bbc-4814-9304-017595651c2e,1,deepseek-ai/deepseek-coder-1.3b-base," - return output_string",93,0.9783 + return output_string",93,0.9783 2025-09-22T14:01:17.542079,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -54647,7 +54647,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return True ",552.1311999982572,,Completions generated successfully. Ready for display in your IDE.,2292ec6e-f0c5-42d3-9ae2-e1cfc9e9bcee,1,deepseek-ai/deepseek-coder-1.3b-base," if not set(dict).issubset(set('')): -",517,0.9872 +",517,0.9872 2025-09-22T14:01:17.701443,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54698,7 +54698,7 @@ class Problem { } }",158.9955999952508,,Completions generated successfully. Ready for display in your IDE.,3a6e8430-4489-45e2-a1de-3a7499140325,1,deepseek-ai/deepseek-coder-1.3b-base," String extension = tokens[1]; -",130,1.0 +",130,1.0 2025-09-22T14:01:17.797840,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54730,7 +54730,7 @@ class Problem { } }",95.43229999690084,,Completions generated successfully. Ready for display in your IDE.,be587301-82c7-4c3d-acb4-1df87e1d0a81,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:17.873738,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54766,7 +54766,7 @@ class Problem { } return (lst1.stream().allMatch(x -> x % 2 == 0)) ? ""YES"" : ""NO""; }",,76.02820000465726,,Completions generated successfully. Ready for display in your IDE.,db7b3b87-f94e-4474-8bae-1aaea9241737,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:01:17.960515,[1],"from typing import List def specialFilter(nums: List[int]) -> int: @@ -54788,7 +54788,7 @@ def specialFilter(nums: List[int]) -> int: return count ",86.77629999874625,,Completions generated successfully. Ready for display in your IDE.,c03cc0f1-dd53-4bbf-8503-f6f2bce86ed1,1,deepseek-ai/deepseek-coder-1.3b-base," continue -",64,1.0 +",64,1.0 2025-09-22T14:01:18.035057,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -54855,7 +54855,7 @@ function do_algebra(operator, operand){ return answer; } ",73.77830000041286,,Completions generated successfully. Ready for display in your IDE.,a2fcc115-2116-45de-ac2d-9c29d774a2a0,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:01:18.161881,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -54876,7 +54876,7 @@ function circular_shift(x, shift){ } ",127.08540000312496,,Completions generated successfully. Ready for display in your IDE.,c11547d8-ce78-4f2f-93e8-eb2e92bf8bc5,1,deepseek-ai/deepseek-coder-1.3b-base," if (shift >= len){ -",103,0.9438 +",103,0.9438 2025-09-22T14:01:18.477905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54920,7 +54920,7 @@ class Problem { } }",315.8402000044589,,Completions generated successfully. Ready for display in your IDE.,1e5c4088-eae7-4009-bd9c-95d348971406,1,deepseek-ai/deepseek-coder-1.3b-base," return nums.stream().filter(x -> x > 10).filter(x -> { -",288,1.0 +",288,1.0 2025-09-22T14:01:18.757226,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54985,7 +54985,7 @@ class Problem { } }",278.8460000010673,,Completions generated successfully. Ready for display in your IDE.,ccdf22f4-4e2e-4e61-b387-ade16a177025,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < grades.size(); ++i) { -",248,1.0 +",248,1.0 2025-09-22T14:01:19.454565,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -55010,7 +55010,7 @@ function odd_count(lst){ // console.log(""The sum is: "", odd_sum) var odd_word = 'the number of odd elements ' + odd_sum + ' n the st'+ String(i+1).padStart(2,'r')+ - 'ring ' + odd",673,0.9015 + 'ring ' + odd",673,0.9015 2025-09-22T14:01:19.517911,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55053,7 +55053,7 @@ class Problem { }).count(); } }",62.973100000817794,,Completions generated successfully. Ready for display in your IDE.,15a51906-150f-490c-8890-974595ee0323,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:01:19.653105,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55095,7 +55095,7 @@ class Problem { if (c1 == 0 || c2 == 0) return ""Yes"";"," } }",135.2365999991889,,Completions generated successfully. Ready for display in your IDE.,66ef6456-217c-4c14-b68e-c8a2a5088c2a,1,deepseek-ai/deepseek-coder-1.3b-base," else return ""No""; -",110,1.0 +",110,1.0 2025-09-22T14:01:19.800773,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -55172,7 +55172,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",147.25569999427535,,Completions generated successfully. Ready for display in your IDE.,91c3f032-ee3a-4ccf-8dce-f3dc031d78df,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",118,1.0 +",118,1.0 2025-09-22T14:01:19.990046,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55212,7 +55212,7 @@ class Problem { } }",189.11250000383006,,Completions generated successfully. Ready for display in your IDE.,6e98ee83-8069-4f8e-815b-03167e514379,1,deepseek-ai/deepseek-coder-1.3b-base," return o1.compareTo(o2); -",161,1.0 +",161,1.0 2025-09-22T14:01:20.215083,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -55238,7 +55238,7 @@ function make_palindrome(string){ } ",224.65650000231108,,Completions generated successfully. Ready for display in your IDE.,55928360-644b-4795-994c-f4a974492cfb,1,deepseek-ai/deepseek-coder-1.3b-base," //find length of string var string_length = string.length; -",186,0.9462 +",186,0.9462 2025-09-22T14:01:20.343542,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55274,7 +55274,7 @@ class Problem { } }",128.3391000033589,,Completions generated successfully. Ready for display in your IDE.,eb60d554-f56b-4faf-80d7-fa717f797a1a,1,deepseek-ai/deepseek-coder-1.3b-base," max = val; -",100,1.0 +",100,1.0 2025-09-22T14:01:20.452390,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55298,4 +55298,4 @@ class Problem { }"," } }",108.67159999907017,,Completions generated successfully. Ready for display in your IDE.,469a9cd6-3163-4746-be40-e9a07caacc4a,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",84,1.0 +",84,1.0 diff --git a/src/performance_and_analysis/chat_benchmark_results.csv b/src/performance_and_analysis/chat_benchmark_results.csv index 409ed6a..2e378a8 100644 --- a/src/performance_and_analysis/chat_benchmark_results.csv +++ b/src/performance_and_analysis/chat_benchmark_results.csv @@ -1,4 +1,4 @@ -timestamp,model_ids,user_prompt,e2e_time_ms,error,chat_id,response_title,assistant_completion,generation_time_ms,confidence +timestamp,model_ids,user_prompt,e2e_time_ms,error,chat_id,response_title,assistant_completion,generation_time_ms,confidence 2025-09-22T22:26:11.629984,[3],Please explain the different components of the solution for the following problem: Write a python function to remove first and last occurrence of a given character from the string.,8808.68640000699,,60ca99b0-e605-4353-b786-6e00c99f0142,Remove first and last occurrence of a given character from a string," The solution involves creating a function that takes three parameters: the input string, the character to be removed, and an optional parameter indicating whether to remove only the first or last occurrence. The function will then iterate through the string and check if the character matches the target character. If it does, the function will either remove the first or last occurrence based on the optional parameter. @@ -14,7 +14,7 @@ Here's a step-by-step breakdown of the solution: Here's the implementation of the function: ```python -def remove_occurrences(string, char, first_last",8765,0.0 +def remove_occurrences(string, char, first_last",8765,0.0 2025-09-22T22:26:20.424422,[3],Please explain the different components of the solution for the following problem: Write a function to sort a given matrix in ascending order according to the sum of its rows.,8794.24740000104,,008a15b0-11b3-47ce-96c6-d46d13bf312d,Sorting Matrix by Row Sums," To solve this problem, we need to follow these steps: @@ -51,7 +51,7 @@ print(sorted_matrix) This will output: ``` -[[4, 5], [7,",8765,0.0 +[[4, 5], [7,",8765,0.0 2025-09-22T22:26:29.212510,[3],Please explain the different components of the solution for the following problem: Write a function to count the most common words in a dictionary.,8787.216400000034,,6d748fdd-d630-427b-9f16-92989fcef0bf,Components of Solution," The solution involves several steps: @@ -82,7 +82,7 @@ def count_most_common_words(text): # Step 4: Find the most common word(s) max_frequency = max(word_count.values()) - most_common_words = [word for word, freq in",8761,0.0 + most_common_words = [word for word, freq in",8761,0.0 2025-09-22T22:26:38.060337,[3],Please explain the different components of the solution for the following problem: Write a python function to find the volume of a triangular prism.,8848.084400000516,,1024d15e-2b04-41d1-902f-e5d8ff648f11,Components of Volume Calculation Function," The volume V of a triangular prism can be calculated using the formula: @@ -115,7 +115,7 @@ def calculate_volume(base_area, height): # Example usage base_area = 50 # Example base area -height",8762,0.0 +height",8762,0.0 2025-09-22T22:26:44.096993,[3],Please explain the different components of the solution for the following problem: Write a function to split a string at lowercase letters.,6035.633899999084,,cf49956b-3b61-4bda-b2e0-45c03d9f4068,String splitting by lowercase letters," The solution involves creating a function that iterates through each character in the string and checks if it is a lowercase letter. If it is, the function splits the string at that point. @@ -130,7 +130,7 @@ def split_at_lowercase(s): print(split_at_lowercase(""HelloWorld"")) # Output: ['H', 'e', 'l', 'l', 'oW', 'o', 'r', 'l', 'd'] ``` -This function works by first creating a new string containing only non-lowercase characters from the input string `s`. It then uses this new string as a delimiter to split the original string into a list of substrings.",6008,0.0 +This function works by first creating a new string containing only non-lowercase characters from the input string `s`. It then uses this new string as a delimiter to split the original string into a list of substrings.",6008,0.0 2025-09-22T22:26:52.885290,[3],Please explain the different components of the solution for the following problem: Write a function to find sequences of lowercase letters joined with an underscore.,8788.721399992937,,81afabc6-7022-455a-8a22-215b0d9b9c30,Components of sequence finding function," The solution involves creating a function that can identify sequences of lowercase letters joined by underscores. Here are the key components: @@ -169,7 +169,7 @@ Putting it all together: import re def find_sequences(input_string): - pattern = r'[a-z]+_' # Regex pattern to",8761,0.0 + pattern = r'[a-z]+_' # Regex pattern to",8761,0.0 2025-09-22T22:26:58.818521,[3],Please explain the different components of the solution for the following problem: Write a function to find the perimeter of a square.,5932.357300000149,,6bbf6d1b-6706-4bfd-b19a-ef5b56bca6f5,Components of Square Perimeter Function," To solve this problem, we need to create a function that calculates the perimeter of a square given its side length. Here are the main components of the solution: @@ -188,7 +188,7 @@ def calculate_perimeter(side_length): return perimeter ``` -This function can then be used by calling it with the appropriate side length as an argument.",5904,0.0 +This function can then be used by calling it with the appropriate side length as an argument.",5904,0.0 2025-09-22T22:27:05.822433,[3],Please explain the different components of the solution for the following problem: Write a function to remove characters from the first string which are present in the second string.,7004.094399992027,,c547acdf-c954-4954-9cc2-95fd50718b98,Function to Remove Characters," The solution involves creating a function that takes two strings as input. The function will iterate through each character in the first string and check if it exists in the second string. If the character does not exist in the second string, it will be added to a new string. Finally, the function will return this new string. @@ -209,7 +209,7 @@ str2 = ""world"" print(remove_chars(str1, str2)) # Output: ""helo"" ``` -This function works by iterating over each character in `str1` and checking if it is not in `str2`. If it is not found in `str2`, it is appended to the `result` string.",6971,0.0 +This function works by iterating over each character in `str1` and checking if it is not in `str2`. If it is not found in `str2`, it is appended to the `result` string.",6971,0.0 2025-09-22T22:27:14.518648,[3],Please explain the different components of the solution for the following problem: Write a function to find whether a given array of integers contains any duplicate element.,8695.679800002836,,c6222eb0-ecb5-46c6-9ca0-94339309152e,Explanation of Solution Components," The solution can be implemented using various approaches. Here are some key components that could be part of the solution: @@ -222,7 +222,7 @@ The solution can be implemented using various approaches. Here are some key comp 4. **Brute Force**: Iterating through each element and checking against all other elements. This method has a time complexity of O(n^2) and a space complexity of O(1). -Each component has its own advantages and trade-offs based on the specific requirements of the problem, such as the size of the input array and available memory.",8659,0.0 +Each component has its own advantages and trade-offs based on the specific requirements of the problem, such as the size of the input array and available memory.",8659,0.0 2025-09-22T22:27:23.306783,[3],Please explain the different components of the solution for the following problem: Write a function to check if the given number is woodball or not.,8788.101400001324,,942c2233-b670-4404-b6a5-aced841c89d8,Woodball Check Function Components," The problem requires us to create a function that checks whether a given number is a ""woodball"" or not. A ""woodball"" can be defined as a number that has at least one digit that is divisible by 5. @@ -251,7 +251,7 @@ def is_woodball(number): #### 2. Digit Extraction To extract each digit from the number, we can convert the number to a string and then iterate over its characters. -```",8761,0.0 +```",8761,0.0 2025-09-22T22:27:32.095961,[3],Please explain the different components of the solution for the following problem: Write a function to find m number of multiples of n.,8788.61859999597,,88968076-413a-4f0e-8501-4a334f1a85db,Components of Solution," To solve this problem, we need to create a function that generates `m` multiples of `n`. Here are the key components of the solution: @@ -276,7 +276,7 @@ def find_multiples(n, m): print(find_multiples(3, 5)) # Output: [3, 6, 9, 12, 15] ``` -This function will generate and return a list containing the first `m` multiples of `",8759,0.0 +This function will generate and return a list containing the first `m` multiples of `",8759,0.0 2025-09-22T22:27:40.885350,[3],Please explain the different components of the solution for the following problem: Write a function to find the first duplicate element in a given array of integers.,8789.134299993748,,b3a9a338-fad1-4201-a86f-fbe4e7065c43,Components of Solution," To solve this problem, we need to create a function that can efficiently find the first duplicate element in an array of integers. Here are the main components of the solution: @@ -305,7 +305,7 @@ seen = {} ``` ### Step 3: Iteration -Traverse through the array and check if each element is in the `seen` dictionary",8762,0.0 +Traverse through the array and check if each element is in the `seen` dictionary",8762,0.0 2025-09-22T22:27:49.696782,[3],Please explain the different components of the solution for the following problem: Write a python function to find the maximum sum of elements of list in a list of lists.,8811.211300009745,,5392571b-5012-49a7-81d3-77a4d9eb5a73,Components of Solution," To solve this problem, we need to create a Python function that can iterate through each sublist in a list of lists and calculate the sum of its elements. The function should then return the maximum sum found among all the sublists. @@ -331,7 +331,7 @@ def max_sum_of_sublists(lst): # Update the maximum sum if the current sum is greater if current_sum > max_sum: - max_sum = current_sum",8764,0.0 + max_sum = current_sum",8764,0.0 2025-09-22T22:27:58.527944,[3],Please explain the different components of the solution for the following problem: Write a function to convert the given binary number to its decimal equivalent.,8830.868400007603,,e5a652b7-bf34-46c5-83c2-f185aa61a1be,Binary to Decimal Conversion," The different components of the solution for converting a binary number to its decimal equivalent include: @@ -364,7 +364,7 @@ binary_number = ""1101"" print(f""The decimal equivalent of {binary_number} is {binary_to_decimal(binary_number)}"") ``` -This function takes a",8760,0.0 +This function takes a",8760,0.0 2025-09-22T22:28:07.321105,[3],Please explain the different components of the solution for the following problem: Write a python function to find the product of non-repeated elements in a given array.,8793.208600007347,,8f62e1f6-1a48-42cb-ae12-2ff851429b8d,Components of Solution," To solve this problem, we need to identify the unique (non-repeated) elements in the given array and then compute their product. @@ -396,7 +396,7 @@ Using the counts obtained from step 1, filter out elements that have a count of ```python def get_non_repeated_elements(element_count): - return [key for key",8765,0.0 + return [key for key",8765,0.0 2025-09-22T22:28:16.110599,[3],Please explain the different components of the solution for the following problem: Write a function to check if the given tuple list has all k elements.,8788.770800005295,,178bc4b3-feb5-4a9f-b048-37ad5e6fdacc,Tuple List Checker," The solution involves creating a function that iterates through each tuple in the list and checks if it contains exactly k elements. Here are the steps: @@ -427,7 +427,7 @@ print(check_tuple_list([(1, 2), (3, 4, 5), (6)], 2)) # Output: [True, False, False] ``` -This will print `[True",8761,0.0 +This will print `[True",8761,0.0 2025-09-22T22:28:24.899060,[3],Please explain the different components of the solution for the following problem: Write a python function to remove all digits from a list of strings.,8787.727999995695,,5c67795a-e562-44f6-8cf5-d776279694bb,Remove Digits from List of Strings," To solve this problem, we need to create a Python function that takes a list of strings and removes all digits from each string in the list. @@ -452,7 +452,7 @@ def remove_digits(strings): return result # Example usage -strings = [""hello123"", ""world456"", ""test789",8760,0.0 +strings = [""hello123"", ""world456"", ""test789",8760,0.0 2025-09-22T22:28:31.961089,[3],Please explain the different components of the solution for the following problem: Write a python function to find binomial co-efficient.,7061.871699988842,,7541d35e-aeeb-4f15-bd31-499fe7d18246,Binomial Coefficient Calculation," The binomial coefficient (also known as ""n choose k"") can be calculated using various methods. Here are some common approaches: @@ -467,7 +467,7 @@ The binomial coefficient (also known as ""n choose k"") can be calculated using 5. **Using Libraries**: Some libraries like NumPy or SciPy offer built-in functions to compute the binomial coefficient efficiently. -Each of these methods has its own advantages and trade-offs depending on the specific requirements and constraints of your application.",7036,0.0 +Each of these methods has its own advantages and trade-offs depending on the specific requirements and constraints of your application.",7036,0.0 2025-09-22T22:28:40.745944,[3],Please explain the different components of the solution for the following problem: Write a python function to find the element occurring odd number of times.,8785.22749998956,,6386609a-ae8d-4d00-a746-b7dfc20419d1,Odd Occurrences Element Finder," The solution involves creating a function that takes a list of elements as input and returns the element that occurs an odd number of times. Here are the steps to achieve this: @@ -499,7 +499,7 @@ def find_odd_occurrence(elements): if value % 2!= 0: return key - #",8760,0.0 + #",8760,0.0 2025-09-22T22:28:49.536590,[3],Please explain the different components of the solution for the following problem: Write a python function to count all the substrings starting and ending with same characters.,8790.105299995048,,8fd9aed0-fa4a-43db-af34-288ada9f6bf6,Substring counting function explanation," The solution involves creating a function that iterates through each character in the string and checks if there is a substring starting and ending with the same character. The function will then increment a counter for every valid substring found. @@ -528,7 +528,7 @@ In this function: - For each substring, we check if its first and last characters are the same. - If they are, we increment the `count`. -This approach ensures that all possible substrings are considered, and the function returns the total count of those that meet the criteria",8764,0.0 +This approach ensures that all possible substrings are considered, and the function returns the total count of those that meet the criteria",8764,0.0 2025-09-22T22:28:58.334995,[3],Please explain the different components of the solution for the following problem: Write a function to find the top k integers that occur most frequently from given lists of sorted and distinct integers using heap queue algorithm.,8797.53019999771,,fa45ffab-1ceb-48e1-ad0d-65260ca1ced2,Heap Queue Algorithm for Top K Integers," To solve this problem, we can utilize the heap queue (priority queue) data structure provided by Python's `heapq` module. Here’s how we can break down the solution into its components: @@ -554,7 +554,7 @@ def find_top_k_frequent_integers(lists): min_heap = [] # Step 3: Frequency Counting - for num in combined",8769,0.0 + for num in combined",8769,0.0 2025-09-22T22:29:07.125139,[3],Please explain the different components of the solution for the following problem: Write a python function to find the largest prime factor of a given number.,8789.93330000958,,b758ec42-b063-4b76-b327-34dbef9439c2,Prime Factorization Function," The solution involves creating a function that takes an integer input and returns its largest prime factor. Here’s how we can break down the steps: @@ -582,7 +582,7 @@ def largest_prime_factor(n): n //= 2 # Check for odd factors from 3 upwards - factor",8760,0.0 + factor",8760,0.0 2025-09-22T22:29:15.674357,[3],Please explain the different components of the solution for the following problem: Write a python function to convert a decimal number to binary number.,8549.057600001106,,422c014c-b062-4ce5-b86a-7d201c80eeee,Convert Decimal to Binary," The different components of the solution include: @@ -606,7 +606,7 @@ def decimal_to_binary(decimal_number): return binary_representation ``` -This function first checks if the input is valid (non-negative integer). Then, it uses a loop to convert the decimal number to binary by repeatedly dividing the number by 2 and appending the remainder to the result string. Finally, it returns the binary representation as a string.",8517,0.0 +This function first checks if the input is valid (non-negative integer). Then, it uses a loop to convert the decimal number to binary by repeatedly dividing the number by 2 and appending the remainder to the result string. Finally, it returns the binary representation as a string.",8517,0.0 2025-09-22T22:29:24.462130,[3],Please explain the different components of the solution for the following problem: Write a python function to find the missing number in a sorted array.,8787.804300009157,,938b5eab-08de-42c8-a3c4-12407becc2ec,Missing Number Solution Components," The solution involves creating a function that takes an input list of integers and returns the missing number. Here are the main steps: @@ -632,7 +632,7 @@ def find_missing_number(arr): # Step 3: Actual sum calculation actual_sum = sum(arr) - # Step 4: Missing",8760,0.0 + # Step 4: Missing",8760,0.0 2025-09-22T22:29:33.248467,[3],Please explain the different components of the solution for the following problem: Write a function to find the n-th rectangular number.,8786.096600000747,,93ad93af-d8e7-471a-b1f4-f1bd1b7c1f75,Explanation of Rectangular Number Function," The rectangular numbers are those that can be expressed as the product of two integers. The n-th rectangular number is the nth number in this sequence. @@ -659,7 +659,7 @@ def nth_rectangular_number(n): while len(rectangular_numbers) < n: j = i + 1 while j <= i: - product = i * j",8757,0.0 + product = i * j",8757,0.0 2025-09-22T22:29:42.043964,[3],Please explain the different components of the solution for the following problem: Write a python function to find the nth digit in the proper fraction of two given numbers.,8794.595900006243,,dc10335b-cc45-4a3f-b4c2-d9af9d4630bb,Components of Solution," The solution involves creating a function that takes two integers as input and returns the nth digit of the proper fraction formed by dividing these two numbers. @@ -694,7 +694,7 @@ Combining all these steps into one function: ```python def find_nth_digit_in_proper_fraction(num1, num2, n): fraction = calculate_proper_fraction(num1, num2) - string_rep = convert_to",8765,0.0 + string_rep = convert_to",8765,0.0 2025-09-22T22:29:50.835859,[3],Please explain the different components of the solution for the following problem: Write a function to sort a given mixed list of integers and strings.,8792.172000001301,,2c50f42e-3458-4e1d-9bd0-9501e1d205d8,Components of sorting function," The solution involves creating a custom sorting function that can handle both integers and strings. Here are the main components: @@ -726,7 +726,7 @@ def custom_sort(mixed_list): return sorted(mixed_list, key=cmp_to_key(compare)) # Example usage: -mixed_list = [3, 'apple', 1, 'banana', 2, 'cherry']",8762,0.0 +mixed_list = [3, 'apple', 1, 'banana', 2, 'cherry']",8762,0.0 2025-09-22T22:29:59.627569,[3],Please explain the different components of the solution for the following problem: Write a function to find the division of first even and odd number of a given list.,8791.076499997871,,70b6292c-0860-415b-bb43-c75894ae3a22,Components of Solution," The solution involves several steps: @@ -763,7 +763,7 @@ def find_division(lst): # Example usage numbers = [1, 2, 3, 4, 5, 6] -result",8763,0.0 +result",8763,0.0 2025-09-22T22:30:08.428613,[3],Please explain the different components of the solution for the following problem: Write a function to check if the letters of a given string can be rearranged so that two characters that are adjacent to each other are different.,8801.014199998463,,f064df76-d908-439d-98d3-2c055639fa58,Explanation of Solution Components," The solution involves several steps: @@ -798,7 +798,7 @@ print(can_rearrange(""aabbcc"")) # Output: False print(can_rearrange(""abcde"")) # Output: True ``` -This function",8774,0.0 +This function",8774,0.0 2025-09-22T22:30:17.219699,[3],Please explain the different components of the solution for the following problem: Write a function to find frequency of the elements in a given list of lists using collections module.,8791.024099991773,,ef25af39-f834-41d7-bad7-486b7b781daf,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -832,7 +832,7 @@ nested_list = [[1, 2, 2], [3, 3, 3, 4], [4, 4]] print(find_frequencies(nested_list)) ``` -This function will correctly compute the frequency of each",8764,0.0 +This function will correctly compute the frequency of each",8764,0.0 2025-09-22T22:30:25.110393,[3],Please explain the different components of the solution for the following problem: Write a function to filter even numbers using lambda function.,7890.150899998844,,2749d958-880e-48fe-9458-cdc3dcf39210,Filtering Even Numbers with Lambda Function," The solution involves creating a list comprehension or a map function that uses a lambda function to check if each number is even. Here are the steps: @@ -856,7 +856,7 @@ even_numbers = [x for x in numbers if x % 2 == 0] print(even_numbers) ``` -Both methods will give you a list of even numbers from the original list.",7863,0.0 +Both methods will give you a list of even numbers from the original list.",7863,0.0 2025-09-22T22:30:33.898758,[3],Please explain the different components of the solution for the following problem: Write a python function to find the sum of repeated elements in a given array.,8787.998699990567,,bd690403-9750-4999-9728-c8e10d0b70b3,Components of Solution," To solve this problem, we need to create a Python function that finds the sum of repeated elements in an array. Here are the main components of the solution: @@ -889,7 +889,7 @@ def sum_of_repeated_elements(arr): return total_sum # Example usage: -arr =",8763,0.0 +arr =",8763,0.0 2025-09-22T22:30:42.697590,[3],Please explain the different components of the solution for the following problem: Write a function to find sequences of lowercase letters joined with an underscore using regex.,8798.702400003094,,5768e151-7a54-4e3c-87aa-88dbea6be9f1,Regex Solution for Underscore Joined Letters," The solution involves creating a regular expression pattern that matches sequences of lowercase letters followed by underscores. Here’s how you can do it step-by-step: @@ -916,7 +916,7 @@ input_str = ""hello_world_this_is_a_test"" print(find_underscore_joined_sequences(input_str)) ``` -In this example, the function `find_underscore_joined_sequences` takes an input string and returns a list of all sequences of lowercase letters joined by an underscore. The regex pattern `[a-z]+_` ensures that only sequences of one or more lowercase letters followed",8762,0.0 +In this example, the function `find_underscore_joined_sequences` takes an input string and returns a list of all sequences of lowercase letters joined by an underscore. The regex pattern `[a-z]+_` ensures that only sequences of one or more lowercase letters followed",8762,0.0 2025-09-22T22:30:51.483783,[3],Please explain the different components of the solution for the following problem: Write a function that matches a word at the beginning of a string.,8785.730800009333,,38f580d6-3488-4e0d-9987-d44998ca50b8,Components of Solution," To solve this problem, we need to break down the task into several components: @@ -943,7 +943,7 @@ print(starts_with_word(""hello world"", ""hello"")) # Output: True print(starts_with_word(""hello world"", ""world"")) # Output: False ``` -This function works by checking if the length of the word is greater than the length of the main string (in which case it cannot be a valid match), then comparing the first `len(word",8759,0.0 +This function works by checking if the length of the word is greater than the length of the main string (in which case it cannot be a valid match), then comparing the first `len(word",8759,0.0 2025-09-22T22:31:00.277594,[3],Please explain the different components of the solution for the following problem: Write a function to find the gcd of the given array elements.,8793.948800012004,,26b40c05-e56d-48b0-9d4b-72ca7ee35fae,GCD Function Components," The GCD (Greatest Common Divisor) of an array can be found by iteratively applying the Euclidean algorithm to pairs of numbers. Here are the key components of the solution: @@ -960,7 +960,7 @@ Here's a step-by-step breakdown of how these components work together: 1. Define the helper function `gcd(a, b)` which computes the GCD of two integers using the Euclidean algorithm. 2. Implement the main function `array_gcd(arr)` which uses the helper function to compute the GCD of the array elements. -3. Iterate through the array, updating the",8763,0.0 +3. Iterate through the array, updating the",8763,0.0 2025-09-22T22:31:05.499848,[3],Please explain the different components of the solution for the following problem: Write a python function to determine whether all the numbers are different from each other are not.,5221.558200006257,,c6c013a7-3109-496c-b364-8bafba4da48e,Components of Solution," The solution involves creating a Python function that checks if all elements in an input list are unique. Here are the main components: @@ -977,7 +977,7 @@ def check_unique_numbers(numbers): return len(set(numbers)) == len(numbers) ``` -This function takes a list of numbers, converts it to a set, and compares their lengths to determine uniqueness.",5189,0.0 +This function takes a list of numbers, converts it to a set, and compares their lengths to determine uniqueness.",5189,0.0 2025-09-22T22:31:14.288740,[3],Please explain the different components of the solution for the following problem: Write a python function to find the last digit when factorial of a divides factorial of b.,8789.274100010516,,206ec519-499d-4dd9-b67c-d2a87be501cf,Factorial Division Last Digit," The solution involves several steps: @@ -1011,7 +1011,7 @@ def last_digit_fact_division(a, b): return last_digit # Example usage -print(last_digit_fact_division(5, 7)) # Output will be the last digit of 7! / 5",8763,0.0 +print(last_digit_fact_division(5, 7)) # Output will be the last digit of 7! / 5",8763,0.0 2025-09-22T22:31:23.076174,[3],Please explain the different components of the solution for the following problem: Write a python function to set all odd bits of a given number.,8786.091000001761,,064d2f68-64f9-4df2-beff-45fd87b6a445,Explanation of Odd Bit Setting Function," The solution involves creating a function that takes an integer input and sets all its odd bits to 1. Here's how we can do it step by step: @@ -1026,7 +1026,7 @@ The solution involves creating a function that takes an integer input and sets a - For example, to set bits at positions 0, 2, 4,..., 30, we would use `mask = 1 | (1 << 2) | (1 << 4) |...`. 4. **Applying the Mask**: - Apply the mask to the input number using bitwise AND operation. -5. **Returning",8760,0.0 +5. **Returning",8760,0.0 2025-09-22T22:31:30.590521,[3],Please explain the different components of the solution for the following problem: Write a function to extract every first or specified element from a given two-dimensional list.,7514.1988999967,,af1dbe3b-69fa-4183-b527-b628c6edd033,Extract elements from 2D list," The solution can be implemented using nested loops or list comprehension. Here's an example using both methods: @@ -1049,7 +1049,7 @@ def extract_elements_using_list_comprehension(matrix, index): Both functions take a `matrix` (a two-dimensional list) and an `index` as input parameters. The `extract_elements_using_nested_loops` function uses nested loops to iterate through each row and column of the matrix, while the `extract_elements_using_list_comprehension` function uses list comprehension to achieve the same result more concisely. -You can choose either method based on your preference or specific requirements.",7486,0.0 +You can choose either method based on your preference or specific requirements.",7486,0.0 2025-09-22T22:31:39.381780,[3],Please explain the different components of the solution for the following problem: Write a function to find the list with minimum length using lambda function.,8791.582999998354,,3de828f2-e7c7-48a3-b292-8f024ab01cc2,Components of Solution," The solution involves creating a function that takes a list of lists as input and returns the list with the minimum length. This can be achieved by using the `min()` function along with a lambda function to specify the key for comparison. @@ -1074,7 +1074,7 @@ Here’s how you can implement this: def find_min_length_list(lists): return min(lists, key=lambda x: len(x)) -# Example",8760,0.0 +# Example",8760,0.0 2025-09-22T22:31:48.173610,[3],Please explain the different components of the solution for the following problem: Write a function to print check if the triangle is equilateral or not.,8791.481500011287,,b8bb7dfe-bf47-48d3-853e-eee128706757,Triangle Check Function," The function should take three sides as input and return True if they form an equilateral triangle, False otherwise. @@ -1108,7 +1108,7 @@ def is_equilateral(a, b, c): # Example usage: print(is_equilateral(3, 3, 3)) # Output: True -print(is_",8762,0.0 +print(is_",8762,0.0 2025-09-22T22:31:54.108667,[3],Please explain the different components of the solution for the following problem: Write a function to caluclate area of a parallelogram.,5934.686499997042,,36930c46-d5fa-4040-94ee-433bc64c43d8,Components of Parallelogram Area Calculation Function," The different components of the solution for calculating the area of a parallelogram include: @@ -1126,7 +1126,7 @@ def calculate_parallelogram_area(base, height): return base * height ``` -This function will correctly compute the area of a parallelogram given valid inputs.",5907,0.0 +This function will correctly compute the area of a parallelogram given valid inputs.",5907,0.0 2025-09-22T22:32:02.660510,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the first and last characters of a given string are equal or not.,8551.682899997104,,33a0b5e3-9c61-4b78-8aee-2dbd5270212d,Components of Solution," The solution involves creating a Python function that takes a string as input and checks if its first and last characters are equal. Here are the main components: @@ -1156,7 +1156,7 @@ def check_first_last_char(input_string): return False ``` -This function will correctly determine if the first and last characters of a given string are equal.",8525,0.0 +This function will correctly determine if the first and last characters of a given string are equal.",8525,0.0 2025-09-22T22:32:11.449431,[3],Please explain the different components of the solution for the following problem: Write a function to sort the given array by using counting sort.,8788.623900007224,,0998167e-4566-47ee-a642-7b4f29e2cb2a,Counting Sort Components," Counting sort is an efficient algorithm for sorting integers within a specific range. Here are its key components: @@ -1186,7 +1186,7 @@ def counting_sort(arr): # Step 1: Initialize the frequency array freq = [0] * (max_val - min_val + 1) - # Step 2: Populate the frequency array",8761,0.0 + # Step 2: Populate the frequency array",8761,0.0 2025-09-22T22:32:19.792407,[3],Please explain the different components of the solution for the following problem: Write a function to find t-nth term of geometric series.,8342.65370000503,,cd3bc21c-c83f-44f7-9150-4b0bb5cdce6a,Geometric Series Function Components," The components of the solution for finding the n-th term of a geometric series include: @@ -1216,7 +1216,7 @@ def nth_term_geometric_series(a, r, n): return a * r**(n - 1) ``` -This function takes the initial term `a`, common ratio `r`, and the desired term position `n` as inputs and returns the n-th term using the formula provided above.",8311,0.0 +This function takes the initial term `a`, common ratio `r`, and the desired term position `n` as inputs and returns the n-th term using the formula provided above.",8311,0.0 2025-09-22T22:32:27.238385,[3],Please explain the different components of the solution for the following problem: Write a python function to check if a given number is one less than twice its reverse.,7445.844899993972,,e6776f5c-e90d-4bcc-ba99-52feb9632366,Explanation of Solution Components," The function should take an integer input and return True or False based on the following condition: @@ -1241,7 +1241,7 @@ def is_one_less_than_twice_reverse(n): return result ``` -This function will correctly determine whether the given number satisfies the specified condition.",7420,0.0 +This function will correctly determine whether the given number satisfies the specified condition.",7420,0.0 2025-09-22T22:32:35.064635,[3],Please explain the different components of the solution for the following problem: Write a python function to find the largest number that can be formed with the given digits.,7825.696899992181,,c8b12a44-bfe9-4fd5-b0cd-fc7e58a146c0,Components of Solution," The solution involves several steps: @@ -1269,7 +1269,7 @@ def largest_number(digits): print(largest_number(['3', '4', '5'])) # Output: ""543"" ``` -This function takes a list of digit strings, converts them to integers, sorts them in descending order, and then joins them back together to form the largest possible number.",7798,0.0 +This function takes a list of digit strings, converts them to integers, sorts them in descending order, and then joins them back together to form the largest possible number.",7798,0.0 2025-09-22T22:32:43.855965,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given two integers have opposite sign or not.,8790.95700000471,,f70f04ea-a83d-47fc-8a4a-f15fe1b14d5d,Opposite Sign Check Function," To solve this problem, we need to create a Python function that checks if two given integers have opposite signs. Here are the steps to achieve this: @@ -1296,7 +1296,7 @@ def check_opposite_sign(num1, num2): # Test cases print(check_opposite_sign(5, -3)) # Output: True -print(check_opposite_sign(-4, 7)) # Output: True",8763,0.0 +print(check_opposite_sign(-4, 7)) # Output: True",8763,0.0 2025-09-22T22:32:52.641006,[3],Please explain the different components of the solution for the following problem: Write a function to find the nth octagonal number.,8784.65440000582,,714f3804-20fb-4982-b9ee-998e0a58f659,Octagonal Number Calculation," The formula for the nth octagonal number is given by: @@ -1325,7 +1325,7 @@ print(octagonal_number(2)) print(octagonal_number(3)) ``` -This will output the first three octagonal numbers respectively",8759,0.0 +This will output the first three octagonal numbers respectively",8759,0.0 2025-09-22T22:33:01.432144,[3],Please explain the different components of the solution for the following problem: Write a function to find the maximum length of the subsequence with difference between adjacent elements for the given array.,8790.779799994198,,62148ec2-88e2-49c4-a539-b8126460f538,Explanation of Solution Components," The solution involves several steps to solve the problem efficiently. Here’s an overview of the key components: @@ -1352,7 +1352,7 @@ def max_length_subsequence(arr): for i in range(1, len(arr)): if abs(arr[i] - arr[i - 1]) <= 1: - curr_len +=",8766,0.0 + curr_len +=",8766,0.0 2025-09-22T22:33:10.224064,[3],Please explain the different components of the solution for the following problem: Write a python function to count number of substrings with the sum of digits equal to their length.,8792.019500004244,,3d31d340-d04b-4bc7-8b9c-acba0a3193ba,Counting Substrings with Sum Equal to Length," To solve this problem, we need to break down the task into several steps: @@ -1387,7 +1387,7 @@ def count_substrings_with_equal_length_and_digit_sum(s): # Example usage input_string = ""abc12"" result = count_substrings_with_equal_length_and_digit_sum(input_string) -print(f""Number of valid",8765,0.0 +print(f""Number of valid",8765,0.0 2025-09-22T22:33:16.771677,[3],Please explain the different components of the solution for the following problem: Write a python function to find smallest number in a list.,6547.512299992377,,b60f570e-ea51-48d2-9ff2-d425e44c99db,Components of Solution," The components of the solution include: @@ -1413,7 +1413,7 @@ def find_smallest_number(numbers): return smallest ``` -This function will correctly identify and return the smallest number from the given list.",6523,0.0 +This function will correctly identify and return the smallest number from the given list.",6523,0.0 2025-09-22T22:33:25.563613,[3],Please explain the different components of the solution for the following problem: Write a function to find the maximum difference between available pairs in the given tuple list.,8791.630299994722,,0e2ad5ff-b21e-421f-b5f9-4a1f4b413b58,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -1454,7 +1454,7 @@ def calculate_differences(pairs): #### Maximum Difference Identification ```python -def find",8763,0.0 +def find",8763,0.0 2025-09-22T22:33:31.016639,[3],Please explain the different components of the solution for the following problem: Write a function to sort a list of tuples using lambda.,5453.054500001599,,05a38f41-8f0e-4ed6-8602-715ac1deff6c,Sorting Tuples with Lambda," The solution involves creating a sorting function that utilizes the `sorted()` function along with a lambda function. The lambda function will specify how to compare the elements during the sorting process. @@ -1470,7 +1470,7 @@ tuples_list = [(1, 'a'), (2, 'b'), (3, 'c')] print(sort_tuples(tuples_list)) ``` -In this example, the lambda function `lambda x: x[0]` tells the `sorted()` function to sort the tuples based on their first element.",5427,0.0 +In this example, the lambda function `lambda x: x[0]` tells the `sorted()` function to sort the tuples based on their first element.",5427,0.0 2025-09-22T22:33:39.801561,[3],Please explain the different components of the solution for the following problem: Write a function of recursion list sum.,8784.25030001381,,57ddf55d-e79b-4408-8304-e06e0a040405,Recursion List Sum Components," The function `recursive_list_sum` takes an input list and returns the sum of all elements in that list using recursion. @@ -1498,7 +1498,7 @@ def recursive_list_sum(lst): return lst[0] + recursive_list_sum(lst[1:]) ``` -This function will correctly",8757,0.0 +This function will correctly",8757,0.0 2025-09-22T22:33:47.315199,[3],Please explain the different components of the solution for the following problem: Write a python function to count positive numbers in a list.,7513.608799999929,,73b07030-c2db-4917-ba2d-15d2143cef9d,Components of Solution," The solution involves creating a function that takes a list as input and returns the count of positive numbers within that list. Here are the key components: @@ -1521,7 +1521,7 @@ def count_positive_numbers(numbers): return count ``` -This function will correctly count the number of positive integers in the provided list.",7486,0.0 +This function will correctly count the number of positive integers in the provided list.",7486,0.0 2025-09-22T22:33:56.107243,[3],Please explain the different components of the solution for the following problem: Write a function to find the number of ways to partition a set of bell numbers.,8791.67010000674,,dddb693e-a0be-4e6b-ab4f-7c42932e7413,Bell Numbers Partitioning," The Bell numbers represent the number of ways to partition a set into non-empty subsets. To solve this problem, we need to understand how to generate Bell numbers and then count the partitions. @@ -1552,7 +1552,7 @@ To count the number of ways to partition a set using a given Bell number, we can import math def partition_count(k): - # This function calculates the number of ways to partition k elements into",8764,0.0 + # This function calculates the number of ways to partition k elements into",8764,0.0 2025-09-22T22:34:04.898829,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given array is monotonic or not.,8790.728499996476,,a1d0a241-69b8-45f3-a2b7-8fb22fababae,Monotonic Array Check," The solution involves creating a function that takes an array as input and checks if it is either non-decreasing (increasing) or non-increasing (decreasing). Here are the steps: @@ -1572,7 +1572,7 @@ The solution involves creating a function that takes an array as input and check 5. **Return Result**: - If both `ascending` and `descending` remain False after the loop, return True indicating the array is neither increasing nor decreasing. - - Otherwise, return True if at least one of them is True; otherwise,",8764,0.0 + - Otherwise, return True if at least one of them is True; otherwise,",8764,0.0 2025-09-22T22:34:13.691392,[3],Please explain the different components of the solution for the following problem: Write a function to check whether a list contains the given sublist or not.,8792.26760000165,,100d9164-1852-4113-a65f-4a439b7a1753,List Sublist Checker," The solution involves creating a function that iterates through the main list and checks if any sublist matches the given sublist. Here are the key components: @@ -1597,7 +1597,7 @@ def check_sublist(main_list, sublist): return False # Test cases -print(check_sublist([1, 2, 3, 4], [2, 3])) # Output: True",8761,0.0 +print(check_sublist([1, 2, 3, 4], [2, 3])) # Output: True",8761,0.0 2025-09-22T22:34:22.480347,[3],Please explain the different components of the solution for the following problem: Write a function to find whether all the given tuples have equal length or not.,8789.264499995625,,9fabdd61-abf4-4c14-a9bc-60fb86566d3c,Tuple Length Checker," The different components of the solution include: @@ -1623,7 +1623,7 @@ def check_tuple_lengths(tuples_list): # Example usage print(check_tuple_lengths([(1, 2), (3, 4), (5, 6)])) # Output: True -print(check_tuple_lengths([(1, 2), (3, 4), (5)])) # Output: False",8763,0.0 +print(check_tuple_lengths([(1, 2), (3, 4), (5)])) # Output: False",8763,0.0 2025-09-22T22:34:31.274119,[3],Please explain the different components of the solution for the following problem: Write a function to sort a list of elements using comb sort.,8792.631999996956,,6ccb7110-416b-4f60-b842-3282f83af1bd,Comb Sort Components," Comb Sort is an efficient sorting algorithm that combines the advantages of insertion sort and binary search. Here are its main components: @@ -1652,7 +1652,7 @@ def comb_sort(arr): if arr[i] > arr[j]: arr[i], arr[j] = arr[j], arr[i] - # Insertion sort on the combined elements",8763,0.0 + # Insertion sort on the combined elements",8763,0.0 2025-09-22T22:34:40.066031,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given number can be represented as difference of two squares or not.,8792.397800003528,,17c3ed2c-ef28-4db9-97ff-e9952471b37e,Components of Solution," To solve this problem, we need to understand that a number n can be represented as the difference of two squares if there exist integers x and y such that n = x^2 - y^2. @@ -1677,7 +1677,7 @@ def can_represent_as_difference_of_two_squares(n): return False for x in range(int(math.sqrt(n)) + 1): y = int(math.sqrt(x**2 + n)) - if y * y ==",8766,0.0 + if y * y ==",8766,0.0 2025-09-22T22:34:48.855695,[3],Please explain the different components of the solution for the following problem: Write a function to split the given string with multiple delimiters by using regex.,8789.074000000255,,2e7dfd4b-de29-42d8-9a12-df658278b32e,Regex String Splitter," The solution involves creating a function that uses regular expressions (regex) to split a string based on multiple delimiters. Here’s how you can do it: @@ -1704,7 +1704,7 @@ def split_string_with_regex(input_str, delimiters): # Split the input string based on the pattern result = re.split(pattern, input_str) - return result",8762,0.0 + return result",8762,0.0 2025-09-22T22:34:56.340035,[3],Please explain the different components of the solution for the following problem: Write a function to check whether it follows the sequence given in the patterns array.,7484.385999996448,,f4018c3a-cc91-4f30-bbeb-6116d463d2a1,Explain components of solution," The solution involves creating a function that checks if an input string matches any pattern defined in a list of patterns. Here are the key components: @@ -1728,7 +1728,7 @@ def check_pattern(input_string, patterns): return False ``` -This function will correctly determine if the input string follows any of the sequences provided in the patterns list.",7418,0.0 +This function will correctly determine if the input string follows any of the sequences provided in the patterns list.",7418,0.0 2025-09-22T22:35:05.135671,[3],Please explain the different components of the solution for the following problem: Write a function to find tuples which have all elements divisible by k from the given list of tuples.,8794.921799999429,,47bcbb71-831c-4c91-9a8d-df6012a1fe34,Tuple Divisibility Check," The solution involves creating a function that iterates through each tuple in the input list, checks if all elements within the tuple are divisible by a given number `k`, and returns a new list containing only those tuples that meet this condition. @@ -1752,7 +1752,7 @@ def find_divisible_tuples(tuples_list, k): # Initialize an empty list to store the results result = [] - # Iterate over each tuple in the",8766,0.0 + # Iterate over each tuple in the",8766,0.0 2025-09-22T22:35:13.938868,[3],Please explain the different components of the solution for the following problem: Write a python function to count the number of squares in a rectangle.,8803.012000003946,,b0cece80-4a17-4460-abcb-349a01de637c,Components of Solution," To solve this problem, we need to break down the task into several components: @@ -1783,7 +1783,7 @@ def count_squares_in_rectangle(width, height): print(count_squares_in_rectangle(5, 5)) # Output: 9 ``` -This function takes two parameters, `width` and `height`, checks their validity, calculates the area, determines the maximum possible side length of a square that fits inside the rectangle,",8761,0.0 +This function takes two parameters, `width` and `height`, checks their validity, calculates the area, determines the maximum possible side length of a square that fits inside the rectangle,",8761,0.0 2025-09-22T22:35:22.730626,[3],Please explain the different components of the solution for the following problem: Write a python function to find the difference between sum of even and odd digits.,8791.205500005162,,7fca5c19-4c7c-4760-99cd-747fd7001f48,Difference between sum of even and odd digits," The solution involves creating a function that takes an integer input, extracts its digits, categorizes them into even or odd, sums up each category, and returns the difference. @@ -1810,7 +1810,7 @@ def difference_of_sums(n): # Step 3: Iterate through each character (digit) in the string representation of the number for char in str_n: - digit = int",8763,0.0 + digit = int",8763,0.0 2025-09-22T22:35:31.521597,[3],Please explain the different components of the solution for the following problem: Write a python function to find number of integers with odd number of set bits.,8791.017299998202,,b0ac3017-8551-4b05-ad1d-f670bed61944,Components of Solution," The solution involves creating a function that counts the number of integers within a given range that have an odd number of set bits (1s) in their binary representation. @@ -1840,7 +1840,7 @@ Let's break down these steps into a more detailed plan: Use Python's built-in functions to convert the number to its binary form. 4. **Count Set Bits**: - Use the `bin()`",8762,0.0 + Use the `bin()`",8762,0.0 2025-09-22T22:35:40.313358,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the length of the word is odd or not.,8791.692500002682,,a56ff393-9e3d-4c3e-96f1-098c5cb189e7,Components of Solution," To solve this problem, we need to create a Python function that checks if the length of a given word is odd or even. Here are the main components of the solution: @@ -1868,7 +1868,7 @@ print(check_word_length(""hello"")) # Output: The length of 'hello' is odd. print(check_word_length(""world"")) # Output: The length of 'world' is even. ``` -This function can",8763,0.0 +This function can",8763,0.0 2025-09-22T22:35:47.208888,[3],Please explain the different components of the solution for the following problem: Write a function to find the nth tetrahedral number.,6895.167800001218,,fd482587-7e34-4931-b8fd-b3197bb20419,Tetrahedral Numbers," The tetrahedral numbers are defined by the formula T(n) = (n*(n+1)*(n+2))/6. To write a function to find the nth tetrahedral number, we need to follow these steps: @@ -1889,7 +1889,7 @@ def tetrahedral_number(n): print(tetrahedral_number(5)) ``` -This function will correctly compute the nth tetrahedral number using the given formula.",6867,0.0 +This function will correctly compute the nth tetrahedral number using the given formula.",6867,0.0 2025-09-22T22:35:54.068226,[3],Please explain the different components of the solution for the following problem: Write a function to zip the two given tuples.,6858.65229999763,,e2745a99-3a83-470c-8ceb-d52743df10f4,Zip Tuples Function," The solution involves creating a function that takes two tuples as input and returns a list of tuples containing elements from both input tuples. @@ -1908,7 +1908,7 @@ def zip_tuples(tuple1, tuple2): return zipped ``` -This function first checks if the lengths of the input tuples are equal. If not, it raises a `ValueError`. Then, it uses a list comprehension along with the built-in `zip` function to create a new list of tuples containing corresponding elements from the input tuples.",6831,0.0 +This function first checks if the lengths of the input tuples are equal. If not, it raises a `ValueError`. Then, it uses a list comprehension along with the built-in `zip` function to create a new list of tuples containing corresponding elements from the input tuples.",6831,0.0 2025-09-22T22:36:02.857985,[3],Please explain the different components of the solution for the following problem: Write a function to find the volume of a sphere.,8789.44130000309,,9131c8ed-da5e-4e2d-98ed-7680138e16a2,Sphere Volume Calculation," The formula to calculate the volume of a sphere is V = (4/3) * π * r^3, where r is the radius of the sphere. @@ -1937,7 +1937,7 @@ You can call this function with a specific radius value to get the volume of the print(sphere_volume(5)) # Output will be approximately 523.60 ``` -This function will correctly calculate the volume of any sphere given its radius.",8759,0.0 +This function will correctly calculate the volume of any sphere given its radius.",8759,0.0 2025-09-22T22:36:11.233677,[3],Please explain the different components of the solution for the following problem: Write a python function to find the character made by adding all the characters of the given string.,8375.472399988212,,03f54f2a-c395-4059-99ef-297c048516ee,Components of Solution," The solution involves creating a function that takes a string as input and returns the character formed by summing up the ASCII values of its characters. Here are the key steps: @@ -1963,7 +1963,7 @@ def find_character(input_string): print(find_character(""hello"")) # Outputs 'h' ``` -This function will correctly compute the desired character based on the sum of ASCII values of the input string.",8350,0.0 +This function will correctly compute the desired character based on the sum of ASCII values of the input string.",8350,0.0 2025-09-22T22:36:20.023463,[3],Please explain the different components of the solution for the following problem: Write a function to find the n-th number in newman conway sequence.,8789.89489999367,,c022f02b-deca-406b-95e5-789b62a102e2,Newman Conway Sequence," The Newman-Conway sequence is a sequence of numbers defined by the recurrence relation: @@ -1995,7 +1995,7 @@ def newman_conway(n): memo[x] = helper(helper(x - 1)) + 1 return memo[x] - return helper(n)",8762,0.0 + return helper(n)",8762,0.0 2025-09-22T22:36:28.707122,[3],Please explain the different components of the solution for the following problem: Write a function to find the surface area of a sphere.,8682.82950000139,,30addc72-7938-49df-8252-0c3834a454e8,Sphere Surface Area Calculation," The formula for the surface area (SA) of a sphere with radius r is given by SA = 4 * π * r^2. @@ -2030,7 +2030,7 @@ if __name__ == ""__main__"": print(f""The surface area of a sphere with radius {radius} is {calculate_surface_area(radius)}"") ``` -This function can be used to find the surface area of any sphere by providing the appropriate radius value.",8657,0.0 +This function can be used to find the surface area of any sphere by providing the appropriate radius value.",8657,0.0 2025-09-22T22:36:37.494237,[3],Please explain the different components of the solution for the following problem: Write a function to find nth centered hexagonal number.,8787.44880000886,,3a37ba8b-a4f0-435d-97ca-38ad374a95d1,Hexagonal Number Calculation," The hexagonal numbers are defined by the formula H(n) = 2n^2 - n. To calculate the nth centered hexagonal number, we need to adjust this formula slightly. @@ -2056,7 +2056,7 @@ def hexagonal_number(n): return (n * (n + 1)) // 2 # Test cases -print(hexagonal_number(1)) # Output should be",8759,0.0 +print(hexagonal_number(1)) # Output should be",8759,0.0 2025-09-22T22:36:46.279954,[3],Please explain the different components of the solution for the following problem: Write a function to merge three dictionaries into a single expression.,8785.087099997327,,e7d49343-f86b-4fa0-ad99-c3aabefdabed,Merging Dictionaries," To solve this problem, we can use several approaches depending on the version of Python you're using. @@ -2082,7 +2082,7 @@ print(merged_dict) # Output: {'a': 1, 'b': 2, 'c': 3} ### Using `|` Operator (Python 3.9+) ```python -def merge_dicts(dict",8758,0.0 +def merge_dicts(dict",8758,0.0 2025-09-22T22:36:54.137895,[3],Please explain the different components of the solution for the following problem: Write a function to get the frequency of the elements in a list.,7857.662499998696,,b7ab17d5-b792-47df-9228-523792b8df9b,Frequency Calculation in List," The different components of the solution for calculating the frequency of elements in a list can be broken down into the following steps: @@ -2108,7 +2108,7 @@ def get_frequency(lst): return frequency ``` -This function takes a list `lst` as input and returns a dictionary where keys are the elements from the list and values are their respective frequencies.",7830,0.0 +This function takes a list `lst` as input and returns a dictionary where keys are the elements from the list and values are their respective frequencies.",7830,0.0 2025-09-22T22:37:02.926570,[3],Please explain the different components of the solution for the following problem: Write a function to find the closest smaller number than n.,8788.53320000053,,e0262ccf-0984-4e3a-aea1-65d78e5c47dc,Closest Smaller Number Function," The solution involves creating a function that takes an integer `n` as input and returns the largest number less than or equal to `n` which has all its digits smaller than any digit in `n`. Here’s how we can approach this: @@ -2134,7 +2134,7 @@ def closest_smaller_number(n): for i in range(n-1, -1, -1): # Convert current number to string str_i = str(i) - if len(str_i) == length:",8758,0.0 + if len(str_i) == length:",8758,0.0 2025-09-22T22:37:11.333472,[3],Please explain the different components of the solution for the following problem: Write a python function to find the length of the longest word.,8406.998999998905,,6027b4e7-f83f-4c07-afd1-6917d8d52322,Components of Longest Word Length Function," The different components of the solution for finding the length of the longest word include: @@ -2166,7 +2166,7 @@ input_string = ""This is a sample sentence"" print(find_longest_word_length(input_string)) ``` -In this example, `find_longest_word_length` takes an input string, splits it into words, calculates their lengths, compares them to find the longest one, and returns its length.",8381,0.0 +In this example, `find_longest_word_length` takes an input string, splits it into words, calculates their lengths, compares them to find the longest one, and returns its length.",8381,0.0 2025-09-22T22:37:20.130061,[3],Please explain the different components of the solution for the following problem: Write a function to check if a substring is present in a given list of string values.,8795.946500002174,,aac456bc-11b2-4858-ad9b-6384ba4cdc8f,Components of Solution," The solution involves creating a function that iterates through each element in the provided list and checks if the substring exists within each element. The function should return True if the substring is found in any of the elements, otherwise False. @@ -2189,7 +2189,7 @@ def check_substring(substring, list_of_strings): return False ``` -This function will efficiently check for the presence of a substring in a list of strings.",8764,0.0 +This function will efficiently check for the presence of a substring in a list of strings.",8764,0.0 2025-09-22T22:37:28.918806,[3],Please explain the different components of the solution for the following problem: Write a function to check whether the given number is undulating or not.,8788.044300003094,,dd415da9-e6d7-4cdf-a9e9-35091dc952c5,Components of Solution," To determine if a number is undulating, we need to evaluate its digits' behavior relative to their neighbors. Here’s how you can break down the solution into components: @@ -2222,7 +2222,7 @@ Compare each digit with its neighbors to determine if the sequence forms an undu ```python def is_undulating(digits): for i in range(1, len(digits) - 1): - if (digits[i] > digits[i -",8762,0.0 + if (digits[i] > digits[i -",8762,0.0 2025-09-22T22:37:37.710247,[3],Please explain the different components of the solution for the following problem: Write a function to calculate the value of 'a' to the power 'b'.,8790.960500002257,,7713f204-a5d7-4aa5-a565-564d957c8f20,Power Calculation Function," The different components of the solution include: @@ -2246,7 +2246,7 @@ def power(a, b): # Test cases print(power(2, 3)) # Output: 8 -print(power(2, -3)) # Output: 0.",8763,0.0 +print(power(2, -3)) # Output: 0.",8763,0.0 2025-09-22T22:37:46.504017,[3],Please explain the different components of the solution for the following problem: Write a function to extract the index minimum value record from the given tuples.,8794.086699999752,,120da51b-994c-43c1-94e2-9ceff296d349,Extracting Minimum Value Record," To solve this problem, we need to create a function that takes a list of tuples as input and returns the tuple with the minimum value. Here's how we can break down the solution into its components: @@ -2271,7 +2271,7 @@ def extract_minimum_value_record(tuples_list): # Step 3 & 4: Iteration and Comparison for t in tuples_list: - if",8762,0.0 + if",8762,0.0 2025-09-22T22:37:55.291954,[3],Please explain the different components of the solution for the following problem: Write a python function to find the minimum length of sublist.,8787.444000001415,,c6e26558-0c28-4d0f-a2a7-a621e9a31195,Components of Solution for Minimum Length Sublist," The solution involves several steps: @@ -2305,7 +2305,7 @@ def min_length_sublist(lst): # Step 4: Validation if all(elem in current_sub_list for elem in lst): - #",8760,0.0 + #",8760,0.0 2025-09-22T22:38:04.081618,[3],Please explain the different components of the solution for the following problem: Write a python function to find the number of divisors of a given integer.,8789.503699998022,,cac23e5f-1ede-48f4-953a-bc6035bea482,Divisor Function Components," To solve this problem, we need to create a Python function that calculates the number of divisors of a given integer. Here are the key components of the solution: @@ -2329,7 +2329,7 @@ def count_divisors(n): count = 0 # Loop through possible divisors - for i in range(1, int(n**0.5) + 1):",8762,0.0 + for i in range(1, int(n**0.5) + 1):",8762,0.0 2025-09-22T22:38:12.869129,[3],Please explain the different components of the solution for the following problem: Write a function to find frequency count of list of lists.,8787.306199999875,,1eb756b4-5fe3-4f42-b6b4-253e1f06e6cc,Frequency Count of List of Lists," To solve this problem, we need to create a function that can handle nested lists and calculate the frequency count of each element within those lists. @@ -2361,7 +2361,7 @@ def frequency_count_of_lists(lst): flat_list = flatten_list(lst) # Step 2: Count frequencies using Counter - freq_count = Counter(",8761,0.0 + freq_count = Counter(",8761,0.0 2025-09-22T22:38:19.938135,[3],Please explain the different components of the solution for the following problem: Write a function to multiply all the numbers in a list and divide with the length of the list.,7068.734699991182,,03d63806-b82b-4f42-a8fe-d56a956c9155,Multiply and Divide List Numbers," To solve this problem, we need to perform two main operations on the list of numbers: @@ -2389,7 +2389,7 @@ numbers = [1, 2, 3, 4] print(multiply_and_divide(numbers)) # Output: 3.0 ``` -This function handles an empty list by returning `None` and performs the required multiplication and division operations.",7042,0.0 +This function handles an empty list by returning `None` and performs the required multiplication and division operations.",7042,0.0 2025-09-22T22:38:28.071869,[3],Please explain the different components of the solution for the following problem: Write a function to convert the given decimal number to its binary equivalent.,8133.280199996079,,2fb06d83-3c05-4d9f-bc8a-9dfcd0812654,Decimal to Binary Conversion," The solution involves creating a function that takes a decimal number as input and returns its binary equivalent. Here are the main steps: @@ -2418,7 +2418,7 @@ def decimal_to_binary(decimal_number): return binary_representation ``` -You can test this function with various decimal numbers to see how it works.",8105,0.0 +You can test this function with various decimal numbers to see how it works.",8105,0.0 2025-09-22T22:38:36.653422,[3],Please explain the different components of the solution for the following problem: Write a function to find the next smallest palindrome of a specified number.,8581.620400000247,,46a8f8d3-08ce-4b23-b621-828750ce330a,Components of Palindrome Function," The solution involves several key components: @@ -2450,7 +2450,7 @@ def next_smallest_palindrome(n): print(next_smallest_palindrome(10)) # Output: 11 ``` -This approach ensures that the function efficiently finds the next smallest palindrome by checking each subsequent number until a palindrome is encountered.",8555,0.0 +This approach ensures that the function efficiently finds the next smallest palindrome by checking each subsequent number until a palindrome is encountered.",8555,0.0 2025-09-22T22:38:45.267721,[3],Please explain the different components of the solution for the following problem: Write a function to find the kth element in the given array.,8614.021900008083,,cc2fb896-c17d-449e-845c-689bcc65585d,Explanation of Solution Components," The solution can be implemented using various approaches such as sorting, binary search, or using a heap data structure. Here are the main components of the solution: @@ -2473,7 +2473,7 @@ The solution can be implemented using various approaches such as sorting, binary - Handle cases when `k` is larger than the length of the array. - Handle cases when the array contains duplicate values. -Each approach has its own advantages and trade-offs in terms of time complexity and space complexity. The choice of approach depends on the specific requirements and constraints of the problem.",8589,0.0 +Each approach has its own advantages and trade-offs in terms of time complexity and space complexity. The choice of approach depends on the specific requirements and constraints of the problem.",8589,0.0 2025-09-22T22:38:52.233116,[3],Please explain the different components of the solution for the following problem: Write a function to convert snake case string to camel case string.,6964.210300007835,,36699c1a-812b-493f-8db3-cb387a4dc866,Snake Case to Camel Case Conversion," The function should take a string as input and return a string with the first letter capitalized and the rest of the words separated by spaces. @@ -2497,7 +2497,7 @@ def snake_case_to_camel_case(snake_str): This function works by splitting the input string at underscores, capitalizing the first letter of each subsequent word (except the first), and then joining them back together without any separators. Let me know if you have any questions or need further clarification! -```",6936,0.0 +```",6936,0.0 2025-09-22T22:39:01.019179,[3],"Please explain the different components of the solution for the following problem: Write a function to find eulerian number a(n, m).",8786.31700000551,,2f998997-19e6-4ba6-a2cb-cd515fa99243,Eulerian Number Components," The Eulerian number a(n, m) can be calculated using the following steps: @@ -2515,7 +2515,7 @@ def eulerian_number(a, b): if a == 0 or b == 0: return 1 elif a > 0 and b > 0: - return sum(eulerian_number(a-1, b), eulerian_number(a-1, b+",8761,0.0 + return sum(eulerian_number(a-1, b), eulerian_number(a-1, b+",8761,0.0 2025-09-22T22:39:09.811272,[3],Please explain the different components of the solution for the following problem: Write a function to sort each sublist of strings in a given list of lists using lambda function.,8791.95109999273,,b2f50018-1874-4175-8ada-f20a634b7025,Sorting Sublists with Lambda Function," To solve this problem, we need to create a function that will take a list of lists (where each sublist contains strings) and sort each sublist individually using a lambda function. @@ -2537,7 +2537,7 @@ def sort_sublists(lists): return [sorted(sublist, key=lambda x: x) for sublist in lists] # Example usage -sublists = [['banana', 'apple', '",8766,0.0 +sublists = [['banana', 'apple', '",8766,0.0 2025-09-22T22:39:16.634778,[3],Please explain the different components of the solution for the following problem: Write a python function to count true booleans in the given list.,6822.8661000030115,,e44a5bd7-58cc-4788-96aa-b45617b7a153,Count True Booleans in List," The solution involves creating a function that iterates through the list and counts the number of `True` values. Here are the steps: @@ -2566,7 +2566,7 @@ def count_true_booleans(lst): return sum(1 for item in lst if item) ``` -Both implementations will give you the desired result.",6798,0.0 +Both implementations will give you the desired result.",6798,0.0 2025-09-22T22:39:24.661233,[3],Please explain the different components of the solution for the following problem: Write a function to add the given list to the given tuples.,8025.697299992316,,7d8f39d8-0bb3-4f1e-9bff-a43cc55d7a3d,Components of Solution," The solution involves creating a function that takes two parameters: a list and a tuple. The function will iterate over each element in the list and append it to the corresponding position in the tuple. Here are the steps: @@ -2591,7 +2591,7 @@ def add_list_to_tuple(lst, tpl): return tpl ``` -This function assumes that both the list and the tuple have the same length. If they do not, it raises a `ValueError`.",8000,0.0 +This function assumes that both the list and the tuple have the same length. If they do not, it raises a `ValueError`.",8000,0.0 2025-09-22T22:39:33.454927,[3],Please explain the different components of the solution for the following problem: Write a python function to count hexadecimal numbers for a given range.,8794.12510000111,,1686cd4a-6d32-4641-a33a-6acd30888cbc,Components of Hex Count Function," The different components of the solution for counting hexadecimal numbers within a given range include: @@ -2621,7 +2621,7 @@ def count_hex_digits(start, end): print(count_hex_digits(1, 10)) # Output should be the count of hexadecimal digits from 1 to 10 ``` -This function will correctly count all hex",8761,0.0 +This function will correctly count all hex",8761,0.0 2025-09-22T22:39:42.243408,[3],Please explain the different components of the solution for the following problem: Write a function to merge multiple sorted inputs into a single sorted iterator using heap queue algorithm.,8788.167599996086,,9c015f7e-2b55-4c2c-8c4d-fe677b79f297,Merging Sorted Inputs with Heap Queue Algorithm," To solve this problem, we need to create a function that merges multiple sorted inputs into a single sorted iterator. The heap queue (priority queue) algorithm can be used here because it efficiently handles merging operations. @@ -2643,7 +2643,7 @@ Here’s a sample implementation in Python: ```python import heapq -def merge",8763,0.0 +def merge",8763,0.0 2025-09-22T22:39:51.033882,[3],Please explain the different components of the solution for the following problem: Write a python function to find the count of rotations of a binary string with odd value.,8789.735199999996,,62044b0b-8f6e-4af1-a2d3-5e42a5aa2637,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -2669,7 +2669,7 @@ def count_rotations(binary_str): # Step 2: Count rotations count = 0 for i in range(n): - # Step 3: Efficiently check for rotations using bitwise operations",8763,0.0 + # Step 3: Efficiently check for rotations using bitwise operations",8763,0.0 2025-09-22T22:39:59.828327,[3],Please explain the different components of the solution for the following problem: Write a function to extract the ranges that are missing from the given list with the given start range and end range values.,8794.587100011995,,5a65fb17-a320-4b72-992d-84d36791c202,Extracting Missing Ranges," To solve this problem, we need to identify the gaps between consecutive elements in the given list. Here are the steps to achieve this: @@ -2698,4 +2698,4 @@ def extract_missing_ranges(lst, start_range, end_range): prev_end = None missing_ranges = [] - for i in range(len(lst",8769,0.0 + for i in range(len(lst",8769,0.0 diff --git a/src/website/src/components/analytics/AgentResults.css b/src/website/src/components/analytics/AgentResults.css new file mode 100644 index 0000000..e845868 --- /dev/null +++ b/src/website/src/components/analytics/AgentResults.css @@ -0,0 +1,112 @@ +/* Agent A/B results comparison — per-arm bars and metric cards. + * + * Shares the card/metric-row/baseline-tag classes already defined in + * StudyManagement.css; only the agent-specific additions live here so the + * shared stylesheet doesn't have to change. + */ + +.agent-results h4 { + margin-bottom: 12px; +} +.agent-bars { + display: flex; + flex-direction: column; + gap: 14px; + margin-bottom: 20px; +} +.agent-bar-metric { + background-color: var(--secondary-color); + border-radius: 8px; + padding: 10px 12px; +} +.agent-bar-metric-header { + display: flex; + justify-content: space-between; + align-items: baseline; + margin-bottom: 8px; +} +.agent-bar-label { + font-weight: 600; + font-size: 0.9rem; +} +.agent-bar-better { + font-size: 0.75rem; + opacity: 0.6; +} +.agent-bar-row { + display: grid; + grid-template-columns: 140px 1fr 90px; + align-items: center; + gap: 8px; + margin: 4px 0; +} +.agent-bar-arm { + font-size: 0.8rem; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: flex; + align-items: center; + gap: 4px; +} +.agent-bar-track { + background-color: var(--border-color); + border-radius: 4px; + height: 14px; + overflow: hidden; +} +.agent-bar-fill { + height: 100%; + background-color: var(--primary-color); + border-radius: 4px; + min-width: 2px; + transition: width 0.3s ease; +} +.agent-bar-fill.baseline { + background-color: #ff9800; +} +.agent-bar-value { + font-size: 0.8rem; + text-align: right; + font-variant-numeric: tabular-nums; +} +.finish-reasons { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 8px; +} +.finish-reason-chip { + font-size: 0.72rem; + background-color: var(--hover-color); + border-radius: 10px; + padding: 2px 8px; +} + +/* Human-in-the-loop panel: the accept/reject/modify signal on agent proposals. + * Highlighted separately because it is the metric that says whether developers + * actually took the agent's work, rather than just how much it produced. */ +.agent-edit-panel { + background-color: var(--secondary-color); + border-radius: 8px; + padding: 10px 12px; + margin-bottom: 20px; +} +.agent-edit-panel h5 { + margin: 0 0 4px 0; + font-size: 0.9rem; + font-weight: 600; +} +.agent-edit-hint { + font-size: 0.75rem; + opacity: 0.7; + margin: 0 0 10px 0; +} +.agent-runtime-chip { + font-size: 0.7rem; + background-color: var(--hover-color); + border-radius: 10px; + padding: 2px 8px; + margin-left: 6px; + white-space: nowrap; +} diff --git a/src/website/src/components/analytics/AgentResults.js b/src/website/src/components/analytics/AgentResults.js new file mode 100644 index 0000000..d56674d --- /dev/null +++ b/src/website/src/components/analytics/AgentResults.js @@ -0,0 +1,301 @@ +import React from "react"; +import "./AgentResults.css"; + +// Headline metrics rendered as side-by-side comparison bars. `better` documents +// the direction researchers care about (it drives the legend, not the layout). +// +// Shapes match the /api/analytics/studies/{id}/agent-evaluation response, which +// aggregates the unified agent_event table — so an arm running Goose, one +// running Codex, and one running the built-in runtime are all directly +// comparable here. +const BAR_METRICS = [ + { + key: "completion_rate", + label: "Task completion rate", + better: "higher", + format: (v) => `${((v || 0) * 100).toFixed(1)}%`, + }, + { + key: "edit_acceptance_rate", + label: "Edit acceptance rate", + better: "higher", + format: (v) => `${((v || 0) * 100).toFixed(1)}%`, + }, + { + key: "avg_steps", + label: "Avg steps / task", + better: "lower", + format: (v) => (v == null ? "n/a" : v.toFixed(1)), + }, + { + key: "avg_model_latency_ms", + label: "Avg model-call latency", + better: "lower", + format: (v) => (v == null || v === 0 ? "n/a" : `${Math.round(v)}ms`), + }, +]; + +const armKey = (arm) => arm.profile_id || arm.profile_name; + +const ComparisonBars = ({ metric, arms }) => { + const values = arms.map((a) => (a.metrics || {})[metric.key]); + const max = Math.max(...values.filter((v) => v != null), 0); + + return ( +
+
+ {metric.label} + + {metric.better === "higher" ? "↑ better" : "↓ better"} + +
+ {arms.map((arm) => { + const value = (arm.metrics || {})[metric.key]; + const pct = max > 0 && value != null ? (value / max) * 100 : 0; + return ( +
+ + {arm.profile_name} + {arm.is_baseline && Baseline} + +
+
+
+ {metric.format(value)} +
+ ); + })} +
+ ); +}; + +// The accept / reject / modify breakdown. Called out on its own because it is +// the only metric that reports what developers did with the agent's output +// rather than how much output there was — an arm that proposes more edits but +// gets fewer accepted is performing worse, not better. +const EditDecisionPanel = ({ arms }) => { + const anyEdits = arms.some((a) => (a.metrics || {}).total_edits > 0); + if (!anyEdits) return null; + + return ( +
+
Human-in-the-loop decisions
+

+ What developers did with each arm's proposed file edits. Undecided edits + are excluded from the acceptance rate. +

+ + + + + + + + + + + + + {arms.map((arm) => { + const m = arm.metrics || {}; + return ( + + + + + + + + + ); + })} + +
ArmProposedAcceptedRejectedModified firstAcceptance
+ {arm.profile_name} + {arm.is_baseline && ( + Baseline + )} + {m.total_edits || 0}{m.accepted_edits || 0}{m.rejected_edits || 0}{m.modified_edits || 0}{`${((m.edit_acceptance_rate || 0) * 100).toFixed(1)}%`}
+
+ ); +}; + +const Uplift = ({ label, value, lowerIsBetter }) => { + if (value == null) return null; + const isGood = lowerIsBetter ? value <= 0 : value >= 0; + return ( +
+ {label}: + + {value > 0 ? "+" : ""} + {value.toFixed(1)}% + +
+ ); +}; + +const ArmCard = ({ arm }) => { + const m = arm.metrics || {}; + const avgTokens = (total) => + m.total_tasks > 0 ? Math.round((total || 0) / m.total_tasks) : null; + + return ( +
+
+ + {arm.profile_name} + {arm.is_baseline && Baseline} + + + {arm.model} + {arm.framework_version && ( + {arm.framework_version} + )} + +
+ +
+
+ Tasks: + + {m.total_tasks || 0} ({m.completed_tasks || 0} completed) + +
+
+ Completion: + + {`${((m.completion_rate || 0) * 100).toFixed(1)}%`} + +
+
+ Participants: + {m.total_participants || 0} +
+
+ Model calls: + {m.model_calls || 0} +
+
+ Tool calls: + {m.tool_calls || 0} +
+
+ Avg steps: + + {m.avg_steps == null ? "n/a" : m.avg_steps.toFixed(1)} + +
+
+ Avg latency: + + {!m.avg_model_latency_ms + ? "n/a" + : `${Math.round(m.avg_model_latency_ms)}ms`} + +
+
+ Avg tokens (in/out): + + {avgTokens(m.total_input_tokens)?.toLocaleString() ?? "n/a"} /{" "} + {avgTokens(m.total_output_tokens)?.toLocaleString() ?? "n/a"} + +
+
+ Total tokens: + + {(m.total_tokens || 0).toLocaleString()} + +
+
+ Edits accepted: + + {m.accepted_edits || 0} of {m.total_edits || 0} + +
+
+ + {arm.uplift && ( +
+ + + + +
+ )} +
+ ); +}; + +const AgentResults = ({ data, profiles }) => { + if (!data) { + return ( +
+

Agent Results

+

+ Loading agent telemetry… If this persists, no agent data has been + collected for this study's arms yet. +

+
+ ); + } + + const arms = data.results || []; + if (arms.length === 0) { + return ( +
+

Agent Results

+

+ {profiles?.length || 0} agent profile(s) attached, but no agent tasks + ran within the study window yet. +

+
+ ); + } + + const hasAnyTasks = arms.some((a) => (a.metrics || {}).total_tasks > 0); + + return ( +
+

Agent Results

+ {!hasAnyTasks && ( +

+ No agent tasks have run for these arms within the study window yet — + metrics will populate as developers use the agent. +

+ )} + +
+ {BAR_METRICS.map((metric) => ( + + ))} +
+ + + +
+ {arms.map((arm) => ( + + ))} +
+
+ ); +}; + +export default AgentResults; diff --git a/src/website/src/components/analytics/AnalyticsNavigation.js b/src/website/src/components/analytics/AnalyticsNavigation.js index 44acfc0..22c1edd 100644 --- a/src/website/src/components/analytics/AnalyticsNavigation.js +++ b/src/website/src/components/analytics/AnalyticsNavigation.js @@ -45,6 +45,20 @@ const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { description: 'Create and manage server/config modules', adminOnly: true }); + navigationItems.push({ + id: 'agent-profiles', + label: 'Agent Profiles', + icon: '🛠️', + description: 'Agent runtime, provider, tools and policy variants', + adminOnly: true + }); + navigationItems.push({ + id: 'agent-assignments', + label: 'Agent Assignments', + icon: '🎲', + description: 'A/B bucket distribution and manual pins', + adminOnly: true + }); } return ( diff --git a/src/website/src/components/analytics/StudyManagement.js b/src/website/src/components/analytics/StudyManagement.js index 0e09d91..e193ba2 100644 --- a/src/website/src/components/analytics/StudyManagement.js +++ b/src/website/src/components/analytics/StudyManagement.js @@ -1,11 +1,16 @@ import React, { useState, useEffect } from 'react'; -import { getStudies, getStudyEvaluation, getStudyDetails, activateStudy, deactivateStudy, createStudy } from '../../utils/api'; +import { getStudies, getStudyEvaluation, getStudyDetails, activateStudy, deactivateStudy, createStudy, getStudyAgentEvaluation } from '../../utils/api'; +import AgentResults from './AgentResults'; import './StudyManagement.css'; const StudyManagement = ({ user }) => { const [studies, setStudies] = useState([]); const [selectedStudy, setSelectedStudy] = useState(null); const [evaluationData, setEvaluationData] = useState(null); + // Agent A/B arms attached to the selected study, and their per-arm results. + // Both stay empty for completion-only studies, which is the common case. + const [agentEvaluationData, setAgentEvaluationData] = useState(null); + const [agentProfiles, setAgentProfiles] = useState([]); const [studyDetails, setStudyDetails] = useState(null); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); @@ -47,6 +52,21 @@ const StudyManagement = ({ user }) => { console.error("Evaluation fetch error:", err); setEvaluationData(null); } + // Agent arms are evaluated by a separate endpoint, because the metrics are + // entirely different (steps, tool calls, edit acceptance) and a study can + // have completion arms, agent arms, or both. + try { + const agentResponse = await getStudyAgentEvaluation(studyId); + if (agentResponse.ok) { + setAgentEvaluationData(agentResponse.data); + } else { + console.warn("Agent evaluation error:", agentResponse.error); + setAgentEvaluationData(null); + } + } catch (err) { + console.error("Agent evaluation fetch error:", err); + setAgentEvaluationData(null); + } }; const fetchStudyDetails = async (studyId) => { @@ -54,13 +74,17 @@ const StudyManagement = ({ user }) => { const response = await getStudyDetails(studyId); if (response.ok) { setStudyDetails(response.data); + // agent_profiles is empty for completion-only studies. + setAgentProfiles(response.data?.agent_profiles || []); } else { console.warn("Details error:", response.error); setStudyDetails(null); + setAgentProfiles([]); } } catch (err) { console.error("Details fetch error:", err); setStudyDetails(null); + setAgentProfiles([]); } }; @@ -68,6 +92,8 @@ const StudyManagement = ({ user }) => { setSelectedStudy(study); setEvaluationData(null); setStudyDetails(null); + setAgentEvaluationData(null); + setAgentProfiles([]); fetchStudyEvaluation(study.study_id); fetchStudyDetails(study.study_id); }; @@ -350,6 +376,15 @@ const StudyManagement = ({ user }) => {
)} + + {agentEvaluationData?.results?.length > 0 && ( +
+ +
+ )} ) : (
diff --git a/src/website/src/pages/AgentAssignments.css b/src/website/src/pages/AgentAssignments.css new file mode 100644 index 0000000..446ce3f --- /dev/null +++ b/src/website/src/pages/AgentAssignments.css @@ -0,0 +1,244 @@ +.agent-assignments-page { + display: flex; + flex-direction: column; + gap: 20px; +} + +.agent-assignments-header { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: flex-start; +} + +.agent-assignments-header h2 { + color: var(--text-color); + margin: 0; +} + +.agent-assignments-header p { + color: var(--text-secondary); + margin: 8px 0 0; + line-height: 1.5; + max-width: 720px; +} + +.assignment-message { + border-radius: 6px; + padding: 12px 14px; + font-size: 14px; + font-weight: 500; +} + +.assignment-message.error { + color: #b3261e; + background: rgba(179, 38, 30, 0.08); + border: 1px solid rgba(179, 38, 30, 0.25); +} + +.assignment-message.success { + color: #0b6b35; + background: rgba(11, 107, 53, 0.08); + border: 1px solid rgba(11, 107, 53, 0.25); +} + +.distribution-panel, +.pin-form, +.assignments-table-panel { + background-color: var(--card-background); + border: 1px solid var(--border-color); + border-radius: 8px; + box-shadow: 0 2px 8px var(--shadow-color); + padding: 20px; +} + +.distribution-panel h3, +.pin-form h3 { + color: var(--text-color); + margin: 0 0 16px; +} + +.distribution-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: 12px; +} + +.distribution-card { + display: flex; + flex-direction: column; + gap: 4px; + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 6px; + padding: 12px 14px; +} + +.distribution-name { + color: var(--text-color); + font-weight: 700; + font-size: 14px; +} + +.distribution-total { + color: var(--primary-color); + font-size: 24px; + font-weight: 800; + line-height: 1; +} + +.distribution-breakdown { + color: var(--text-secondary); + font-size: 12px; +} + +.pin-form-row { + display: flex; + gap: 16px; + align-items: flex-end; + flex-wrap: wrap; +} + +.pin-form label { + color: var(--text-color); + display: flex; + flex-direction: column; + gap: 6px; + font-size: 14px; + font-weight: 600; + flex: 1 1 200px; +} + +.pin-form input, +.pin-form select, +.assignments-table select { + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 6px; + color: var(--text-color); + font-size: 14px; + padding: 10px 12px; +} + +.pin-form input:focus, +.pin-form select:focus { + border-color: var(--primary-color); + outline: none; + box-shadow: 0 0 0 2px rgba(66, 133, 244, 0.18); +} + +.table-actions { + display: flex; + gap: 8px; + flex-wrap: wrap; + align-items: center; +} + +.primary-button, +.secondary-button, +.danger-button { + border-radius: 6px; + cursor: pointer; + font-size: 14px; + font-weight: 600; + padding: 9px 14px; + transition: all 0.2s ease; +} + +.primary-button { + background-color: var(--primary-color); + border: 1px solid var(--primary-color); + color: white; +} + +.secondary-button { + background-color: var(--secondary-color); + border: 1px solid var(--border-color); + color: var(--text-color); +} + +.danger-button { + background-color: transparent; + border: 1px solid rgba(179, 38, 30, 0.45); + color: #b3261e; +} + +.primary-button:hover, +.secondary-button:hover, +.danger-button:hover { + transform: translateY(-1px); + box-shadow: 0 2px 4px var(--shadow-color); +} + +.primary-button:disabled, +.secondary-button:disabled, +.danger-button:disabled { + cursor: not-allowed; + opacity: 0.65; + transform: none; +} + +.assignments-table-panel { + overflow-x: auto; + padding: 0; +} + +.assignments-table { + border-collapse: collapse; + min-width: 720px; + width: 100%; +} + +.assignments-table th, +.assignments-table td { + border-bottom: 1px solid var(--border-color); + color: var(--text-color); + padding: 14px 16px; + text-align: left; + vertical-align: middle; +} + +.assignments-table th { + background-color: var(--secondary-color); + color: var(--text-secondary); + font-size: 12px; + font-weight: 700; + text-transform: uppercase; +} + +.assignments-table tbody tr:hover { + background-color: var(--hover-color); +} + +.source-badge { + border-radius: 999px; + font-size: 12px; + font-weight: 700; + padding: 3px 10px; + text-transform: capitalize; +} + +.source-badge.auto { + color: #0b6b35; + background: rgba(11, 107, 53, 0.1); + border: 1px solid rgba(11, 107, 53, 0.25); +} + +.source-badge.manual { + color: #92500a; + background: rgba(146, 80, 10, 0.1); + border: 1px solid rgba(146, 80, 10, 0.3); +} + +.assignments-empty { + color: var(--text-secondary); + line-height: 1.5; + padding: 28px; + text-align: center; +} + +@media (max-width: 900px) { + .agent-assignments-header { + flex-direction: column; + } +} diff --git a/src/website/src/pages/AgentAssignments.js b/src/website/src/pages/AgentAssignments.js new file mode 100644 index 0000000..f2b3715 --- /dev/null +++ b/src/website/src/pages/AgentAssignments.js @@ -0,0 +1,297 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { + deleteAgentAssignment, + getAgentAssignments, + getAgentProfiles, + setAgentAssignment, +} from "../utils/api"; +import "./AgentAssignments.css"; + +const formatDate = (value) => { + if (!value) return "—"; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toLocaleString(); +}; + +const shortId = (id) => (id ? `${String(id).slice(0, 8)}…` : "—"); + +const AgentAssignments = () => { + const [assignments, setAssignments] = useState([]); + const [profiles, setProfiles] = useState([]); + const [pinUserId, setPinUserId] = useState(""); + const [pinProfileId, setPinProfileId] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); + + const profileNameById = useMemo(() => { + const map = {}; + profiles.forEach((p) => { + map[p.profile_id || p.id] = p.name; + }); + return map; + }, [profiles]); + + // Which runtime each arm targets. Worth surfacing here because arms in the + // same study can now run entirely different agents (Goose vs Codex vs the + // built-in runtime), and that is usually the variable under test. + const frameworkByName = useMemo(() => { + const map = {}; + profiles.forEach((p) => { + if (p.name) map[p.name] = p.framework_version; + }); + return map; + }, [profiles]); + + // Count of users assigned to each profile, split by how they got there. + const distribution = useMemo(() => { + const counts = {}; + assignments.forEach((a) => { + const key = a.profile_name || profileNameById[a.profile_id] || "unknown"; + if (!counts[key]) counts[key] = { auto: 0, manual: 0, total: 0 }; + counts[key][a.source === "manual" ? "manual" : "auto"] += 1; + counts[key].total += 1; + }); + return Object.entries(counts).sort((a, b) => a[0].localeCompare(b[0])); + }, [assignments, profileNameById]); + + const loadData = async () => { + setIsLoading(true); + setError(""); + const [assignmentsRes, profilesRes] = await Promise.all([ + getAgentAssignments(), + getAgentProfiles(), + ]); + if (assignmentsRes.ok) { + setAssignments(assignmentsRes.data || []); + } else { + setError(assignmentsRes.error); + } + if (profilesRes.ok) { + setProfiles( + Array.isArray(profilesRes.data) ? profilesRes.data : [], + ); + } + setIsLoading(false); + }; + + useEffect(() => { + loadData(); + }, []); + + const handlePin = async (event) => { + event.preventDefault(); + if (!pinUserId.trim()) { + setError("User ID is required."); + return; + } + if (!pinProfileId) { + setError("Pick a profile to pin the user to."); + return; + } + + setIsSaving(true); + setError(""); + setNotice(""); + const response = await setAgentAssignment(pinUserId.trim(), pinProfileId); + if (response.ok) { + setNotice("User pinned to profile."); + setPinUserId(""); + setPinProfileId(""); + await loadData(); + } else { + setError(response.error); + } + setIsSaving(false); + }; + + const handleReassign = async (userId, profileId) => { + if (!profileId) return; + setIsSaving(true); + setError(""); + setNotice(""); + const response = await setAgentAssignment(userId, profileId); + if (response.ok) { + setNotice("Assignment updated (manual)."); + await loadData(); + } else { + setError(response.error); + } + setIsSaving(false); + }; + + const handleClear = async (userId) => { + const confirmed = window.confirm( + "Clear this assignment? The user will be re-bucketed at random on their next agent task.", + ); + if (!confirmed) return; + + setIsSaving(true); + setError(""); + setNotice(""); + const response = await deleteAgentAssignment(userId); + if (response.ok) { + setNotice("Assignment cleared."); + await loadData(); + } else { + setError(response.error); + } + setIsSaving(false); + }; + + return ( +
+
+
+

Agent Assignments

+

+ Which A/B variant each user is pinned to. Assignments are normally + drawn at random on a user's first agent task; pin a user manually + only for test accounts or debugging — manual pins are excluded from + A/B analysis. +

+
+ +
+ + {(error || notice) && ( +
+ {error || notice} +
+ )} + +
+

Bucket distribution

+ {distribution.length === 0 ? ( +
No assignments yet.
+ ) : ( +
+ {distribution.map(([name, counts]) => ( +
+ {name} + {counts.total} + + {counts.auto} auto · {counts.manual} manual + {frameworkByName[name] ? ` · ${frameworkByName[name]}` : ""} + +
+ ))} +
+ )} +
+ +
+

Pin a user

+
+ + + +
+
+ +
+ {isLoading ? ( +
Loading assignments…
+ ) : assignments.length === 0 ? ( +
+ No users have been assigned a profile yet. +
+ ) : ( + + + + + + + + + + + + {assignments.map((a) => ( + + + + + + + + ))} + +
UserProfileSourceAssigned atActions
{shortId(a.user_id)} + {a.profile_name || + profileNameById[a.profile_id] || + shortId(a.profile_id)} + + + {a.source} + + {formatDate(a.assigned_at)} +
+ + +
+
+ )} +
+
+ ); +}; + +export default AgentAssignments; diff --git a/src/website/src/pages/AgentProfiles.css b/src/website/src/pages/AgentProfiles.css new file mode 100644 index 0000000..b666bd1 --- /dev/null +++ b/src/website/src/pages/AgentProfiles.css @@ -0,0 +1,316 @@ +.agent-profiles-page { + display: flex; + flex-direction: column; + gap: 20px; +} + +.agent-profiles-header { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: flex-start; +} + +.agent-profiles-header h2, +.profile-form h3 { + color: var(--text-color); + margin: 0; +} + +.agent-profiles-header p { + color: var(--text-secondary); + margin: 8px 0 0; + line-height: 1.5; +} + +.profile-message { + border-radius: 6px; + padding: 12px 14px; + font-size: 14px; + font-weight: 500; +} + +.profile-message.error { + color: #b3261e; + background: rgba(179, 38, 30, 0.08); + border: 1px solid rgba(179, 38, 30, 0.25); +} + +.profile-message.success { + color: #0b6b35; + background: rgba(11, 107, 53, 0.08); + border: 1px solid rgba(11, 107, 53, 0.25); +} + +.agent-profiles-layout { + display: grid; + grid-template-columns: minmax(280px, 340px) 1fr; + gap: 20px; + align-items: start; +} + +.profile-form, +.profiles-table-panel { + background-color: var(--card-background); + border: 1px solid var(--border-color); + border-radius: 8px; + box-shadow: 0 2px 8px var(--shadow-color); +} + +.profile-form { + display: flex; + flex-direction: column; + gap: 16px; + padding: 20px; +} + +.profile-form label { + color: var(--text-color); + display: flex; + flex-direction: column; + gap: 6px; + font-size: 14px; + font-weight: 600; +} + +.profile-form label.checkbox-label { + flex-direction: row; + align-items: center; + gap: 8px; +} + +.profile-form label.checkbox-label input[type="checkbox"] { + width: auto; + padding: 0; + margin: 0; +} + +.profile-form input, +.profile-form select { + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 6px; + color: var(--text-color); + font-size: 14px; + padding: 10px 12px; +} + +.profile-form input:focus, +.profile-form select:focus { + border-color: var(--primary-color); + outline: none; + box-shadow: 0 0 0 2px rgba(66, 133, 244, 0.18); +} + +.profile-form-actions, +.table-actions { + display: flex; + gap: 8px; + flex-wrap: wrap; +} + +.primary-button, +.secondary-button, +.danger-button { + border-radius: 6px; + cursor: pointer; + font-size: 14px; + font-weight: 600; + padding: 9px 14px; + transition: all 0.2s ease; +} + +.primary-button { + background-color: var(--primary-color); + border: 1px solid var(--primary-color); + color: white; +} + +.secondary-button { + background-color: var(--secondary-color); + border: 1px solid var(--border-color); + color: var(--text-color); +} + +.danger-button { + background-color: transparent; + border: 1px solid rgba(179, 38, 30, 0.45); + color: #b3261e; +} + +.primary-button:hover, +.secondary-button:hover, +.danger-button:hover { + transform: translateY(-1px); + box-shadow: 0 2px 4px var(--shadow-color); +} + +.primary-button:disabled, +.secondary-button:disabled, +.danger-button:disabled { + cursor: not-allowed; + opacity: 0.65; + transform: none; +} + +.profiles-table-panel { + overflow-x: auto; +} + +.profiles-table { + border-collapse: collapse; + min-width: 760px; + width: 100%; +} + +.profiles-table th, +.profiles-table td { + border-bottom: 1px solid var(--border-color); + color: var(--text-color); + padding: 14px 16px; + text-align: left; + vertical-align: middle; +} + +.profiles-table th { + background-color: var(--secondary-color); + color: var(--text-secondary); + font-size: 12px; + font-weight: 700; + text-transform: uppercase; +} + +.profiles-table tbody tr:hover { + background-color: var(--hover-color); +} + +.profiles-empty { + color: var(--text-secondary); + line-height: 1.5; + padding: 28px; + text-align: center; +} + +.tools-field { + display: flex; + flex-direction: column; + gap: 6px; +} + +.tools-field-label { + color: var(--text-color); + font-size: 14px; + font-weight: 600; + display: flex; + align-items: center; + gap: 8px; +} + +.tools-count { + color: var(--text-secondary); + font-size: 12px; + font-weight: 400; +} + +.tools-dropdown { + position: relative; +} + +.tools-dropdown-trigger { + width: 100%; + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 6px; + color: var(--text-color); + cursor: pointer; + font-size: 14px; + padding: 10px 12px; + text-align: left; +} + +.tools-dropdown-trigger:focus { + border-color: var(--primary-color); + outline: none; + box-shadow: 0 0 0 2px rgba(66, 133, 244, 0.18); +} + +.tools-dropdown-trigger:disabled { + cursor: not-allowed; + opacity: 0.65; +} + +.tools-dropdown-summary { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.tools-dropdown-caret { + flex: 0 0 auto; + color: var(--text-secondary); + font-size: 12px; +} + +.tools-dropdown-menu { + position: absolute; + top: calc(100% + 4px); + left: 0; + right: 0; + z-index: 20; + max-height: 240px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 6px; + background-color: var(--card-background); + border: 1px solid var(--border-color); + border-radius: 6px; + box-shadow: 0 4px 12px var(--shadow-color); + padding: 10px 12px; +} + +.tools-select-all { + font-weight: 600; +} + +.tools-dropdown-divider { + border-top: 1px solid var(--border-color); + margin: 2px 0; +} + +.profile-form label.tool-checkbox-label { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-start; + gap: 6px; + min-width: 0; + font-size: 13px; + font-weight: 400; + color: var(--text-color); + cursor: pointer; + overflow-wrap: anywhere; + word-break: break-word; +} + +.tool-checkbox-label input[type="checkbox"] { + flex: 0 0 auto; + width: auto; + padding: 0; + margin: 0; + cursor: pointer; +} + +@media (max-width: 900px) { + .agent-profiles-layout { + grid-template-columns: 1fr; + } + + .agent-profiles-header { + flex-direction: column; + } +} diff --git a/src/website/src/pages/AgentProfiles.js b/src/website/src/pages/AgentProfiles.js new file mode 100644 index 0000000..3b1db0f --- /dev/null +++ b/src/website/src/pages/AgentProfiles.js @@ -0,0 +1,628 @@ +import React, { useEffect, useMemo, useRef, useState } from "react"; +import { + createAgentProfile, + deleteAgentProfile, + getAgentAvailableTools, + getAgentProfiles, + updateAgentProfile, +} from "../utils/api"; +import "./AgentProfiles.css"; + +// Runtimes a profile can target. Must stay in sync with SUPPORTED_FRAMEWORKS in +// backend/routers/agent/profiles.py, which rejects anything else. +const FRAMEWORKS = [ + { + value: "code4me2-agent", + label: "code4me2-agent (built-in)", + hint: "Our own ReAct loop. Self-reports telemetry; tools below are enforced by the runtime.", + }, + { + value: "goose", + label: "Goose", + hint: "Third-party ACP agent. Observed via the plugin proxy; tools below act as an allowlist filter.", + }, + { + value: "codex", + label: "Codex", + hint: "Third-party ACP agent on OpenAI's Responses API. Manages its own tools — the selection below is ignored.", + }, +]; + +const APPROVAL_POLICIES = [ + { value: "per_step", label: "Ask per step" }, + { value: "suggestion_only", label: "Suggestion only (never applies edits)" }, + { value: "auto", label: "Auto-approve" }, +]; + +const EMPTY_FORM = { + name: "", + model: "", + framework_version: "code4me2-agent", + base_url: "", + api_key_ref: "", + tools: [], + approval_policy: "per_step", + max_steps: 15, + max_context_tokens: "", + is_active: true, + temperature: "", // blank = provider default (no override) +}; + +const TEMPERATURE_MIN = 0; +const TEMPERATURE_MAX = 2; + +const formatTools = (toolsJson) => { + try { + const parsed = JSON.parse(toolsJson || "[]"); + return Array.isArray(parsed) ? parsed.join(", ") : ""; + } catch (_) { + return toolsJson || ""; + } +}; + +const getProfileId = (profile) => profile.profile_id || profile.id || profile.name; + +const AgentProfiles = () => { + const [profiles, setProfiles] = useState([]); + const [availableTools, setAvailableTools] = useState([]); + const [form, setForm] = useState(EMPTY_FORM); + const [editingProfileId, setEditingProfileId] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); + const [isToolsOpen, setIsToolsOpen] = useState(false); + const toolsDropdownRef = useRef(null); + + const sortedProfiles = useMemo( + () => [...profiles].sort((a, b) => (a.name || "").localeCompare(b.name || "")), + [profiles], + ); + + const loadProfiles = async () => { + setIsLoading(true); + setError(""); + const response = await getAgentProfiles(); + if (response.ok) { + setProfiles(Array.isArray(response.data) ? response.data : []); + } else { + setError(response.error); + } + setIsLoading(false); + }; + + useEffect(() => { + loadProfiles(); + }, []); + + // The selectable tool set depends on the runtime, so reload it whenever the + // runtime changes rather than showing tools the agent can't actually call. + useEffect(() => { + let cancelled = false; + getAgentAvailableTools(form.framework_version).then((res) => { + if (!cancelled && res.ok) setAvailableTools(res.data.tools || []); + }); + return () => { + cancelled = true; + }; + }, [form.framework_version]); + + useEffect(() => { + if (!isToolsOpen) return undefined; + const handleClickOutside = (event) => { + if ( + toolsDropdownRef.current && + !toolsDropdownRef.current.contains(event.target) + ) { + setIsToolsOpen(false); + } + }; + document.addEventListener("mousedown", handleClickOutside); + return () => document.removeEventListener("mousedown", handleClickOutside); + }, [isToolsOpen]); + + const validateForm = () => { + if (!form.name.trim()) return "Profile name is required."; + if (!/^[a-z0-9_-]+$/.test(form.name.trim())) { + return "Use lowercase letters, numbers, hyphens, or underscores for the name."; + } + if (!form.model.trim()) return "Model is required."; + if (form.temperature !== "" && form.temperature !== null) { + const t = Number(form.temperature); + if (Number.isNaN(t) || t < TEMPERATURE_MIN || t > TEMPERATURE_MAX) { + return "Temperature must be a number between 0 and 2 (or blank)."; + } + } + if (form.base_url.trim() && !/^https?:\/\//i.test(form.base_url.trim())) { + return "Base URL must start with http:// or https://"; + } + // Guard against the obvious mistake of pasting the key itself. The backend + // rejects this too; catching it here avoids a round-trip and, more + // importantly, avoids sending a real credential over the wire at all. + const keyRef = form.api_key_ref.trim(); + if (keyRef && (keyRef.length > 128 || /[-.\s/:]/.test(keyRef))) { + return "API key ref must be the NAME of an environment variable (e.g. OPENAI_API_KEY), not the key itself."; + } + return ""; + }; + + const resetForm = () => { + setForm(EMPTY_FORM); + setEditingProfileId(null); + setError(""); + setNotice(""); + }; + + const handleChange = (event) => { + const { name, value, type, checked } = event.target; + setForm((current) => ({ + ...current, + [name]: type === "checkbox" ? checked : value, + })); + }; + + const handleTemperatureChange = (event) => { + const { value } = event.target; + if (value === "") { + setForm((current) => ({ ...current, temperature: "" })); + return; + } + const numeric = Number(value); + if (Number.isNaN(numeric)) return; + const clamped = Math.min(TEMPERATURE_MAX, Math.max(TEMPERATURE_MIN, numeric)); + setForm((current) => ({ ...current, temperature: clamped })); + }; + + const handleToolToggle = (toolName) => { + setForm((current) => { + const selected = new Set(current.tools); + if (selected.has(toolName)) { + selected.delete(toolName); + } else { + selected.add(toolName); + } + return { ...current, tools: Array.from(selected) }; + }); + }; + + const allToolsSelected = + availableTools.length > 0 && form.tools.length === availableTools.length; + + const handleToggleSelectAllTools = () => { + setForm((current) => ({ + ...current, + tools: allToolsSelected ? [] : [...availableTools], + })); + }; + + const handleSubmit = async (event) => { + event.preventDefault(); + const validationError = validateForm(); + if (validationError) { + setError(validationError); + return; + } + + setIsSaving(true); + setError(""); + setNotice(""); + + const payload = { + name: form.name.trim(), + model: form.model.trim(), + framework_version: form.framework_version, + // Blank means "use the server default upstream", which the backend + // represents as NULL rather than an empty string. + base_url: form.base_url.trim() || null, + api_key_ref: form.api_key_ref.trim() || null, + // The API takes a JSON string, not an array. + tools_json: JSON.stringify(form.tools), + approval_policy: form.approval_policy, + max_steps: Number(form.max_steps), + max_context_tokens: + form.max_context_tokens === "" ? null : Number(form.max_context_tokens), + is_active: Boolean(form.is_active), + temperature: + form.temperature === "" || form.temperature === null + ? null + : Number(form.temperature), + }; + + const response = editingProfileId + ? await updateAgentProfile(editingProfileId, payload) + : await createAgentProfile(payload); + + if (response.ok) { + setNotice(editingProfileId ? "Agent profile updated." : "Agent profile created."); + resetForm(); + await loadProfiles(); + } else { + setError(response.error); + } + + setIsSaving(false); + }; + + const handleEdit = (profile) => { + setEditingProfileId(getProfileId(profile)); + let selectedTools = []; + try { + const parsed = JSON.parse(profile.tools_json || "[]"); + selectedTools = Array.isArray(parsed) ? parsed : []; + } catch (_) { + selectedTools = []; + } + setForm({ + name: profile.name || "", + model: profile.model || "", + framework_version: profile.framework_version || "code4me2-agent", + base_url: profile.base_url || "", + api_key_ref: profile.api_key_ref || "", + tools: selectedTools, + approval_policy: profile.approval_policy || "per_step", + max_steps: profile.max_steps || 15, + max_context_tokens: + profile.max_context_tokens === null || + profile.max_context_tokens === undefined + ? "" + : profile.max_context_tokens, + is_active: profile.is_active !== undefined ? Boolean(profile.is_active) : true, + temperature: + profile.temperature === null || profile.temperature === undefined + ? "" + : profile.temperature, + }); + setError(""); + setNotice(""); + }; + + const handleDelete = async (profile) => { + const profileId = getProfileId(profile); + const confirmed = window.confirm( + `Delete agent profile "${profile.name}"? Historical agent tasks reference ` + + `profiles by name, so past telemetry stays attributable — but any A/B ` + + `assignments pointing at this profile are removed. To retire an arm ` + + `mid-study, uncheck "Active" instead.`, + ); + if (!confirmed) return; + + setIsSaving(true); + setError(""); + setNotice(""); + + const response = await deleteAgentProfile(profileId); + if (response.ok) { + setNotice("Agent profile deleted."); + if (editingProfileId === profileId) resetForm(); + await loadProfiles(); + } else { + setError(response.error); + } + + setIsSaving(false); + }; + + const selectedSet = new Set(form.tools); + const activeFramework = FRAMEWORKS.find( + (f) => f.value === form.framework_version, + ); + const toolsIgnored = form.framework_version === "codex"; + + return ( +
+
+
+

Agent Profiles

+

+ Define the experiment arms for agent runs: which runtime, which + OpenAI-compatible provider, which model, tools, approval policy and + step limit. A profile's settings are snapshotted onto each task when + it starts, so editing a profile never changes tasks already running. +

+
+ +
+ + {(error || notice) && ( +
+ {error || notice} +
+ )} + +
+
+

{editingProfileId ? "Edit Profile" : "Create Profile"}

+ + + + + {activeFramework && ( +

{activeFramework.hint}

+ )} + + + + +

+ Any endpoint speaking the OpenAI-compatible chat-completions format: + Ollama, Groq, OpenRouter, vLLM, or OpenAI itself. Leave blank to use + the server's configured default. +

+ + +

+ The name of an environment variable on the server — + never the key itself. Keys are never stored in the database. +

+ + + +
+ + Tools + + {form.tools.length} / {availableTools.length} selected + + +
+ + {isToolsOpen && availableTools.length > 0 && ( +
+ +
+ {availableTools.map((tool) => ( + + ))} +
+ )} +
+
+

+ Selecting no tools is a meaningful condition, not an omission: it + produces a genuinely tool-free arm. +

+ + + + + + + + + +
+ + {editingProfileId && ( + + )} +
+ + +
+ {isLoading ? ( +
Loading agent profiles...
+ ) : sortedProfiles.length === 0 ? ( +
+ No agent profiles found. At least one active profile is required + before any agent task can be created. +
+ ) : ( + + + + + + + + + + + + + + + + + {sortedProfiles.map((profile) => ( + + + + + + + + + + + + + ))} + +
NameRuntimeModelProviderApprovalToolsStepsTempActiveActions
{profile.name}{profile.framework_version || "—"}{profile.model}{profile.base_url || "server default"}{profile.approval_policy}{formatTools(profile.tools_json) || "none"}{profile.max_steps} + {profile.temperature === null || + profile.temperature === undefined + ? "default" + : profile.temperature} + {profile.is_active === false ? "No" : "Yes"} +
+ + +
+
+ )} +
+
+
+ ); +}; + +export default AgentProfiles; diff --git a/src/website/src/pages/Dashboard.js b/src/website/src/pages/Dashboard.js index a559016..842e5d8 100644 --- a/src/website/src/pages/Dashboard.js +++ b/src/website/src/pages/Dashboard.js @@ -8,6 +8,8 @@ import ModelAnalytics from "../components/analytics/ModelAnalytics"; import CalibrationAnalytics from "../components/analytics/CalibrationAnalytics"; import StudyManagement from "../components/analytics/StudyManagement"; import ConfigManagement from "../components/analytics/ConfigManagement"; +import AgentProfiles from "./AgentProfiles"; +import AgentAssignments from "./AgentAssignments"; import "./Dashboard.css"; const Dashboard = ({ user, onLogout }) => { @@ -19,6 +21,8 @@ const Dashboard = ({ user, onLogout }) => { if (baseViews.has(view)) return true; if (view === 'studies') return !!user?.is_admin; if (view === 'configs') return !!user?.is_admin; + if (view === 'agent-profiles') return !!user?.is_admin; + if (view === 'agent-assignments') return !!user?.is_admin; return false; }; @@ -84,6 +88,10 @@ const Dashboard = ({ user, onLogout }) => { return ; case 'configs': return ; + case 'agent-profiles': + return ; + case 'agent-assignments': + return ; default: return { return { ok: false, error: "Validation failed" }; } }; + +// ── Agent subsystem ───────────────────────────────────────────────────────── +// +// Backs the three admin pages: AgentProfiles (define experiment arms), +// AgentAssignments (inspect / pin the A/B buckets), and AgentResults +// (compare arms). All endpoints are admin-only server-side. + +const AGENT_BASE = () => + `${process.env.REACT_APP_BACKEND_HOST}:${process.env.REACT_APP_BACKEND_PORT}/api/agent`; + +// Shared request helper. The hand-rolled fetch blocks above predate it; new +// agent endpoints funnel through here so error handling stays consistent. +const agentRequest = async (path, { method = "GET", body, label } = {}) => { + try { + const response = await fetch(`${AGENT_BASE()}${path}`, { + method, + headers: { "Content-Type": "application/json" }, + credentials: "include", + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + // 204 has no body to parse. + const payload = + response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) { + return { + ok: false, + error: + payload["detail"] || + payload["message"] || + `${response.status}: ${response.statusText}`, + }; + } + return { ok: true, data: payload }; + } catch (e) { + console.error(`Error ${label || path}:`, e); + return { ok: false, error: `Failed to ${label || "complete request"}` }; + } +}; + +export const getAgentProfiles = async () => { + const result = await agentRequest("/profiles", { label: "load agent profiles" }); + return result.ok ? { ok: true, data: result.data.profiles || [] } : result; +}; + +export const createAgentProfile = async (profile) => + agentRequest("/profiles", { + method: "POST", + body: profile, + label: "create agent profile", + }); + +export const updateAgentProfile = async (profileId, profile) => + agentRequest(`/profiles/${profileId}`, { + method: "PUT", + body: profile, + label: "update agent profile", + }); + +export const deleteAgentProfile = async (profileId) => + agentRequest(`/profiles/${profileId}`, { + method: "DELETE", + label: "delete agent profile", + }); + +// Tool names selectable for a profile. Pass a frameworkVersion to narrow the +// list to the runtime the profile targets. +export const getAgentAvailableTools = async (frameworkVersion) => { + const query = frameworkVersion + ? `?framework_version=${encodeURIComponent(frameworkVersion)}` + : ""; + const result = await agentRequest(`/available-tools${query}`, { + label: "load available agent tools", + }); + return result.ok + ? { + ok: true, + data: { + tools: result.data.tools || [], + frameworks: result.data.frameworks || [], + }, + } + : result; +}; + +export const getAgentAssignments = async () => { + const result = await agentRequest("/assignments", { + label: "load agent assignments", + }); + return result.ok + ? { ok: true, data: result.data.assignments || [] } + : result; +}; + +// Pin a user to a specific arm. Recorded as source="manual", which excludes +// the row from re-rolls and from A/B analysis. +export const setAgentAssignment = async (userId, profileId) => + agentRequest(`/assignments/${userId}`, { + method: "PUT", + body: { profile_id: profileId }, + label: "set agent assignment", + }); + +// Clear an assignment so the user is re-drawn on their next agent task. +export const deleteAgentAssignment = async (userId) => + agentRequest(`/assignments/${userId}`, { + method: "DELETE", + label: "clear agent assignment", + }); + +// Per-arm comparison for a study's agent arms. +export const getStudyAgentEvaluation = async (studyId) => { + try { + const response = await fetch( + `${process.env.REACT_APP_BACKEND_HOST}:${process.env.REACT_APP_BACKEND_PORT}/api/analytics/studies/${studyId}/agent-evaluation`, + { + method: "GET", + headers: { "Content-Type": "application/json" }, + credentials: "include", + }, + ); + const body = await response.json(); + if (!response.ok) { + return { + ok: false, + error: body["detail"] || `${response.status}: ${response.statusText}`, + }; + } + return { ok: true, data: body }; + } catch (e) { + console.error("Error loading agent evaluation:", e); + return { ok: false, error: "Failed to load agent evaluation" }; + } +}; diff --git a/tests/database_tests/test_migration_manager.py b/tests/database_tests/test_migration_manager.py index 740ac91..540eb73 100644 --- a/tests/database_tests/test_migration_manager.py +++ b/tests/database_tests/test_migration_manager.py @@ -201,13 +201,37 @@ def test_setup_migration_tracking_success(self): result = manager.setup_migration_tracking() assert result + def test_setup_migration_tracking_stamps_base_without_new_revision(self): + """Tracking setup must stamp "base" and NOT generate a revision file. + + Generating a revision here would put a fresh database on a parallel + dead-end branch, so the subsequent `upgrade head` would skip every real + migration — including the one that creates the agent tables. Asserting + on the absence of the `revision` call is what keeps that regression + from coming back silently. + """ + with patch("database.migration.migration_manager.Path") as mock_path, patch( + "database.migration.migration_manager.Config" + ), patch("database.migration.migration_manager.command") as mock_command: + mock_path.return_value.exists.return_value = True + + manager = MigrationManager() + result = manager.setup_migration_tracking() + + assert result + mock_command.stamp.assert_called_once() + assert mock_command.stamp.call_args[0][1] == "base" + mock_command.revision.assert_not_called() + def test_setup_migration_tracking_failure(self): """Test failed migration tracking setup.""" with patch("database.migration.migration_manager.Path") as mock_path, patch( "database.migration.migration_manager.Config" ), patch("database.migration.migration_manager.command") as mock_command: mock_path.return_value.exists.return_value = True - mock_command.revision.side_effect = Exception("Command failed") + # `stamp` is the alembic call the setup now makes (it no longer + # generates a revision), so that's what has to fail here. + mock_command.stamp.side_effect = Exception("Command failed") manager = MigrationManager() result = manager.setup_migration_tracking() From 76b55db8255b4dd616f68dfaa2e732f0f387fc97 Mon Sep 17 00:00:00 2001 From: Roham Koohestani Date: Mon, 7 Sep 2026 10:47:34 +0200 Subject: [PATCH 002/115] [Feature]: Add Docker Compose setup for local development on non-GPU architectures (ARM/CPU). Introduce agent subsystem configuration and update environment handling for CORS and ACP integration. Update README with new agent features and migration instructions. --- .env | 91 -------- .gitignore | 1 + Dockerfile.cpu | 29 +++ README.md | 111 ++++++++- docker-compose.dev-arm.yaml | 217 ++++++++++++++++++ docker-compose.yml | 14 ++ ...2b3c4d5f6a8_seed_default_agent_profiles.py | 24 +- 7 files changed, 385 insertions(+), 102 deletions(-) delete mode 100644 .env create mode 100644 Dockerfile.cpu create mode 100644 docker-compose.dev-arm.yaml diff --git a/.env b/.env deleted file mode 100644 index d10a0df..0000000 --- a/.env +++ /dev/null @@ -1,91 +0,0 @@ -# ----------------------- -# General & Server Settings -# ----------------------- -DATA_DIR=./data - -SERVER_VERSION_ID=1 -SERVER_HOST=0.0.0.0 -SERVER_PORT=8008 - -# ----------------------- -# Authentication & Rate Limits -# ----------------------- -AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=86400 -SESSION_TOKEN_EXPIRES_IN_SECONDS=7200 -EMAIL_VERIFICATION_TOKEN_EXPIRES_IN_SECONDS=86400 -RESET_PASSWORD_TOKEN_EXPIRES_IN_SECONDS=900 -TOKEN_HOOK_ACTIVATION_IN_SECONDS=10 -DEFAULT_MAX_REQUEST_RATE_PER_HOUR=500 -MAX_REQUEST_RATE_PER_HOUR_CONFIG='{"/api/completion/multi_file_context/update":300,"/api/completion/feedback":600,"/api/completion/get":100,"/api/completion/request":600,"/api/project/activate":100,"/api/project/create":100,"/api/session/acquire":100,"/api/user/authenticate":100,"/api/user/create":100,"/api/user/delete":100,"/api/user/update":100,"/api/ws/completion":1200,"/api/ws/completion/multi_file_context":300}' - - -# ----------------------- -# Database Configuration -# ----------------------- -DB_HOST=localhost -DB_PORT=5432 -DB_USER=postgres -DB_PASSWORD=postgres -DB_NAME=code4meV2 - -# ----------------------- -# SQLAlchemy Engine Connection Config -# ----------------------- -DB_POOL_SIZE=20 -DB_MAX_OVERFLOW=30 -DB_POOL_TIMEOUT=5 -DB_POOL_RECYCLE=1800 - -# ----------------------- -# pgAdmin Configuration -# ----------------------- -PGADMIN_HOST=localhost -PGADMIN_PORT=8080 -PGADMIN_DEFAULT_EMAIL=admin@admin.com -PGADMIN_DEFAULT_PASSWORD=admin - -# ----------------------- -# Frontend Configuration -# ----------------------- -WEBSITE_HOST=http://localhost -WEBSITE_PORT=3000 -REACT_APP_GOOGLE_CLIENT_ID=909086079826-skaqesnvhn16q0vq0vn9iot3no8g67h5.apps.googleusercontent.com - -# ----------------------- -# Redis & Celery Configuration -# ----------------------- -REDIS_HOST=localhost -REDIS_PORT=6379 -CELERY_BROKER_HOST=localhost -CELERY_BROKER_PORT=6380 - -# ----------------------- -# Email Configuration -# ----------------------- -EMAIL_HOST=smtp.example.com -EMAIL_PORT=587 -EMAIL_USERNAME=YOUR_EMAIL_USERNAME -EMAIL_PASSWORD=YOUR_EMAIL_PASSWORD -EMAIL_USE_TLS=true -EMAIL_FROM=YOUR_EMAIL -VERIFICATION_URL=http://localhost:8008/api/user/verify -RESET_PASSWORD_URL=http://localhost:8008/api/user/reset-password - -# ----------------------- -# Model Configuration -# ----------------------- -PRELOAD_MODELS=false -MODEL_CACHE_DIR=./.cache -MODEL_USE_CACHE=true -MODEL_USE_COMPILE=true -MODEL_WARMUP=true - -# ----------------------- -# Thread Pool Configuration -# ----------------------- -THREAD_POOL_MAX_WORKERS=2 - -# ----------------------- -# Hugging Face Configuration -HF_TOKEN=your_huggingface_token_here -# ----------------------- \ No newline at end of file diff --git a/.gitignore b/.gitignore index c0569b5..21ab2bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store +.env node_modules .venv .idea diff --git a/Dockerfile.cpu b/Dockerfile.cpu new file mode 100644 index 0000000..d3ab818 --- /dev/null +++ b/Dockerfile.cpu @@ -0,0 +1,29 @@ +# CPU/no-GPU build for local development on machines without an NVIDIA GPU +# (e.g. Apple Silicon / aarch64). The default `Dockerfile` targets a CUDA host +# and its `nvidia-container-toolkit`/`libnvidia-ml-dev` apt packages have no +# arm64 build, so it cannot be built here at all. This variant drops every +# NVIDIA-specific step; `transformers`/`torch` fall back to CPU automatically +# when no GPU is visible, which is all the agent subsystem needs (it talks to +# an external OpenAI-compatible upstream, not a locally loaded model). +# +# Used by docker-compose.dev-arm.yaml, not by the production docker-compose.yml. +FROM continuumio/miniconda3:latest + +WORKDIR /app + +# argon2-cffi (and a few other deps) build C extensions from source on arm64 +# since no prebuilt wheel exists for this platform/Python combo. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libffi-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN conda create -y -n myenv python=3.10 && conda clean -a -y +RUN /bin/bash -c "source activate myenv && conda install -y -c conda-forge huggingface_hub[cli] && conda clean -a -y" + +COPY requirements.txt . +RUN /bin/bash -c "source activate myenv && pip install --no-cache-dir -r requirements.txt" +RUN /bin/bash -c "source activate myenv && pip install uvicorn" + +COPY . . diff --git a/README.md b/README.md index 184d6c3..82f6202 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Docker](https://img.shields.io/badge/docker-ready-blue.svg)](https://docker.com) [![License](https://img.shields.io/badge/License-Apache_2.0-brightgreen.svg)](https://www.apache.org/licenses/LICENSE-2.0) -An advanced AI-powered code completion platform, featuring ghost text suggestions, multi-model inference, a context-aware chat assistant, real-time WebSocket communication, and collaborative development tools suitable for conducting empirical studies on developer behaviour. +An advanced AI-powered code completion platform, featuring ghost text suggestions, multi-model inference, a context-aware chat assistant, an autonomous coding agent (ACP), real-time WebSocket communication, and collaborative development tools suitable for conducting empirical studies on developer behaviour. ## 🎯 Project Overview @@ -14,6 +14,7 @@ Code4me V2 is a research platform that combines transformer models with real-wor - **Real-time Code Completion**: Multi-model AI inference with WebSocket streaming - **Collaborative Development**: Multi-user project management and session tracking +- **Autonomous Coding Agent**: A locally-launched ACP (Agent Client Protocol) agent, with server-assigned model/provider/tools for A/B study arms — see "Agent Subsystem (ACP)" under System Architecture below - **Advanced Analytics**: Comprehensive telemetry and behavioral analysis in a dedicated analysis platform - **Research Platform**: Ground truth collection and model evaluation tools @@ -69,8 +70,16 @@ EMAIL_PORT=587 EMAIL_USERNAME=your_email@gmail.com EMAIL_PASSWORD=your_app_password EMAIL_FROM=noreply@code4me.com + +# Agent Subsystem (ACP) — upstream provider for the coding agent. +# Defaults to a local Ollama instance (no key, no cost). See .env.example +# for the full list of agent variables and the "Agent Subsystem" section below. +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 +AGENT_UPSTREAM_API_KEY= ``` +See [`.env.example`](.env.example) for the complete, documented set of agent-specific variables (per-provider API keys, Codex reasoning effort override, etc.). + ### 3. Deploy with Docker ```bash # Start all services @@ -120,6 +129,15 @@ docker-compose ps - **Project Chat System**: `/api/ws/chat` - Collaborative chat functionality - **Multi-file Context**: `/api/ws/multi-file-context` - Context updates +#### 🤖 Agent Subsystem (ACP) +The `backend` service is also a relay + telemetry sink for an autonomous coding agent (it does not run agent inference loops itself): + +- **Third-party agents** (e.g. Goose, Codex) run inside the IDE plugin process and call back through `POST /api/agent/inference`, which the backend proxies to an OpenAI-compatible upstream (Ollama, OpenAI, Groq, OpenRouter, or any compatible endpoint). +- **The built-in `code4me2-agent`** runs as a separate local OS process, launched by the IDE plugin, speaking ACP over stdio. It authenticates via a grant → session handoff: the plugin calls `POST /api/acp/grant`, the agent process exchanges it for a bearer token at `POST /api/acp/session/exchange`, then fetches its assigned model/provider/tools from `GET /api/acp/agent-config`. +- An **agent profile** (`agent_profile` table) defines a runtime + provider + model + tools + approval policy — used as an A/B study arm. An **agent assignment** is a sticky, server-authoritative per-user draw; the client never self-selects. +- Provider API keys are never stored in the database — a profile stores only the *name* of an environment variable (`api_key_ref`), resolved from the backend's own environment at request time. See [`.env.example`](.env.example). +- Installing and running the local `code4me2-agent` CLI is a separate, standalone step — see "Running the Agent CLI (`code4me2-agent`)" under Development below. + #### 📊 Analytics & Telemetry - **Behavioral Analytics**: Typing patterns, acceptance rates, interaction timings - **Performance Metrics**: Model response times, accuracy measurements @@ -174,6 +192,36 @@ GET /api/chat/get/{page_number} # Paginated chat history DELETE /api/chat/delete/{chat_id} # Delete chat session ``` +### Agent Subsystem (ACP) +```http +# Profiles & assignments (admin auth) +GET /api/agent/profiles # List agent profiles +POST /api/agent/profiles # Create agent profile +GET /api/agent/available-tools # List tools an adapter can expose +GET /api/agent/registry # List available adapter frameworks +GET /api/agent/assignments # List per-user profile assignments + +# Self-report ingestion & memory (ACP bearer auth, from the local agent process) +POST /api/agent/events/ingest # Ingest agent run/edit telemetry +GET /api/agent/runs/{run_id} # Fetch a stored agent run +GET /api/agent/memory/{session_id} # Read persisted agent memory +PUT /api/agent/memory/{session_id} # Update persisted agent memory +DELETE /api/agent/memory/{session_id} # Delete persisted agent memory + +# Task lifecycle & inference relay (plugin session-cookie auth) +POST /api/agent/task # Start an agent task +POST /api/agent/task/{task_id}/close # Close an agent task +POST /api/agent/task/{task_id}/telemetry # Report task-level telemetry +POST /api/agent/inference # Relay a chat-completion call to the assigned upstream + +# ACP grant handoff (bootstraps the locally launched agent process) +POST /api/acp/grant # Issue a single-use grant (plugin -> backend) +POST /api/acp/session/exchange # Exchange a grant for a bearer session +POST /api/acp/session/validate-or-refresh # Validate/refresh an ACP session +GET /api/acp/agent-config # Fetch assigned model/provider/tools +POST /api/acp/persistent-auth-token # Issue a long-lived token for unattended runs (admin only) +``` + ### Real-time WebSocket Endpoints ```http WS /api/ws/completion # Real-time completion streaming @@ -207,6 +255,7 @@ curl -s \ - **Completions**: AI-generated suggestions with performance metrics - **Telemetry**: Comprehensive usage analytics for research - **Context**: Multi-file code context with vector embeddings +- **Agent tables**: `agent_profile`, `agent_profile_assignment`, `agent_task`, `agent_event`, `agent_edit`, `agent_memory`, `study_agent_profile` — coding agent A/B profiles, per-user assignments, task lifecycle, and self-reported telemetry/edits ### Key Features - **PostgreSQL + pgvector**: Vector similarity search for code context @@ -214,6 +263,22 @@ curl -s \ - **Migration System**: Hybrid SQL initialization + Alembic migrations - **Comprehensive Indexing**: Optimized for high-volume analytics queries +### Running Migrations +The agent tables (and any future schema change) are applied via the hybrid migration manager, not automatically on startup: +```bash +# First-time setup (initializes from init.sql + stamps Alembic tracking) +python src/database/migration/migration_manager.py init + +# Apply pending migrations (e.g. the agent tables + seeded default profiles) +python src/database/migration/migration_manager.py migrate + +# Check current revision / history +python src/database/migration/migration_manager.py status +``` +The seed migration (`e2b3c4d5f6a8_seed_default_agent_profiles`) only writes `api_key_ref` *names* (e.g. `OPENROUTER_API_KEY`) into the database — it never reads or requires those environment variables at migration time. Only `default-code4me2-agent` is seeded active by default; the Goose/Codex profiles are seeded inactive until you provide those external binaries. + +**Current default profile (initial testing config)**: `default-code4me2-agent` points at OpenRouter (`https://openrouter.ai/api/v1`) using the free `cohere/north-mini-code:free` model, with `api_key_ref=OPENROUTER_API_KEY`. Set `OPENROUTER_API_KEY` in `.env` once you have a key — until then, requests are forwarded unauthenticated and OpenRouter will 401. Update the profile via `PUT /api/agent/profiles/{profile_id}` (or the admin UI) to switch models/providers later. + ## ⚡ Asynchronous Processing ### Celery Task Queues @@ -306,6 +371,24 @@ Notes: - The API runs on `SERVER_HOST`/`SERVER_PORT` (default `0.0.0.0:8008`). - When running via Docker, `nginx` serves the website on `http://localhost:8000` and proxies `/api/*` to the backend. +### Running the Agent CLI (`code4me2-agent`) +`code4me2-agent` is a **separate, standalone package** (`pyproject.toml` at the repo root) — it is not installed into the backend's `requirements.txt`/Docker image, since it runs as its own local OS process launched by the JetBrains plugin (or manually, for development), not inside the FastAPI server. + +```bash +# Install the agent CLI package (from the repo root, ideally in its own venv) +pip install -e . + +# Write default config files to ~/.code4me/ and print IDE setup instructions +code4me2-agent --setup + +# Run it directly over stdio (normally the IDE plugin does this for you) +code4me2-agent +``` +Notes: +- The agent has **no hardcoded model/provider/API key** — after it authenticates via the ACP grant handoff (`POST /api/acp/grant` → `POST /api/acp/session/exchange`), it fetches `GET /api/acp/agent-config` and the backend's assigned agent profile overrides its local config. This is deliberate: a stale local config can't silently override a study assignment. +- The seeded default profile currently points at OpenRouter (`cohere/north-mini-code:free`) — set `OPENROUTER_API_KEY` in `.env` before it will authenticate. If you instead assign a local-Ollama profile, make sure `ollama serve` is running before launching the agent. +- Local-process environment variables (set by whatever launches the agent, e.g. the plugin): `CODE4ME_ACP_BACKEND_URL`, `CODE4ME_ACP_GRANT`, `CODE4ME_ACP_TOKEN`, `CODE4ME_ACP_LOG_SECRETS`, `CODE4ME_AGENT_LOG_LEVEL`, `CODE4ME_MODEL_REQUEST_LIMIT`, `CODE4ME_MODEL_REQUEST_WINDOW_SECONDS`, `CODE4ME_BACKEND_429_MAX_RETRIES`, `CODE4ME_BACKEND_429_RETRY_SLEEP`, `CODE4ME_RATE_LIMIT_STATE_PATH`. These are distinct from the backend's own `AGENT_*`/`*_API_KEY` variables in `.env`. + ### Frontend Development ```bash cd src/website @@ -340,9 +423,18 @@ src/ main.py # FastAPI entrypoint (serves /api, docs at /docs) App.py # Application singleton (DB, Redis, Celery, Models) backend/routers/ # REST and WS route modules (mounted at /api) + agent/ # Agent profiles, consent, ingest, memory (/api/agent) + agents.py # Agent task lifecycle + inference relay (/api/agent) + acp/ # ACP grant handoff (/api/acp) celery_app/ # Celery setup and task modules + agents/ # Backend-side agent logic: provider routing, registry, + # normalization, event/telemetry ingestion (imported + # by backend/routers/agent* — not a Celery task) + code4me2_agent/ # Standalone ACP agent CLI (separate package, see + # pyproject.toml — not part of the backend image) database/ # SQLAlchemy models, CRUD, migrations, pgvector website/ # React frontend (served via nginx at :8000) + src/pages/AgentProfiles.js, AgentAssignments.js # Admin-only agent management UI ``` ## 📊 Monitoring & Health Checks @@ -397,6 +489,14 @@ docker-compose logs -f celery-worker **Common 401/403 Causes** - Missing `session_token`/`project_token` cookies when calling protected endpoints. - Using POST for `/api/session/acquire` (it is GET). +- Agent endpoints under `/api/agent/events`, `/api/agent/memory` require an ACP bearer token (from `/api/acp/session/exchange`), not a session cookie. +- `/api/agent/profiles` and `/api/agent/assignments` require an admin user. + +**Agent Subsystem Issues** +- **401 from OpenRouter**: the seeded default profile (`default-code4me2-agent`) uses OpenRouter with `api_key_ref=OPENROUTER_API_KEY`. Set `OPENROUTER_API_KEY` in `.env` and restart `backend` — until it's set, calls are forwarded unauthenticated and OpenRouter rejects them. +- **Agent can't reach Ollama from inside Docker** (only relevant if you switch a profile back to a local Ollama `base_url`): `http://localhost:11434/v1` resolves to the *container*, not your host. Either run the backend outside Docker for agent development, or set that profile's `base_url` (or the `AGENT_UPSTREAM_BASE_URL` fallback) to `http://host.docker.internal:11434/v1` (the `backend` service already maps `host.docker.internal` via `extra_hosts` in `docker-compose.yml`). +- **Agent tables missing / seed profiles absent**: run `python src/database/migration/migration_manager.py migrate` — the agent tables are not created by `init.sql`, only by Alembic migrations. +- **Agent CLI won't authenticate**: check that the launching process (normally the JetBrains plugin) set `CODE4ME_ACP_BACKEND_URL`, `CODE4ME_ACP_GRANT`, and `CODE4ME_ACP_TOKEN`, or that the workspace handoff file exists — the grant is single-use and short-lived. ### Debugging Commands ```bash @@ -445,6 +545,15 @@ MODEL_CACHE_DIR=./data/hf MODEL_MAX_NEW_TOKENS=64 MODEL_USE_CACHE=true +# Agent Subsystem (ACP) — see .env.example for the full documented list +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 +AGENT_UPSTREAM_API_KEY= +OLLAMA_API_KEY= +OPENAI_API_KEY= +GROQ_API_KEY= +OPENROUTER_API_KEY= +AGENT_REASONING_EFFORT= + # Security Configuration AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=3600 SESSION_TOKEN_EXPIRES_IN_SECONDS=3600 diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml new file mode 100644 index 0000000..bacd21e --- /dev/null +++ b/docker-compose.dev-arm.yaml @@ -0,0 +1,217 @@ +# Local dev stack for machines without an NVIDIA GPU (Apple Silicon / aarch64, +# or any x86_64 box without CUDA). Mirrors docker-compose.yml but: +# - builds backend/celery-worker from Dockerfile.cpu instead of Dockerfile +# - drops GPU device reservations, NVIDIA_*/CUDA_* env vars, and the NVML +# library bind-mounts +# - defaults PRELOAD_MODELS=false, since loading a multi-GB transformer +# model on CPU is impractical for local testing and unnecessary for +# exercising the agent subsystem (which calls an external OpenAI-compatible +# upstream such as OpenRouter, not a locally loaded model) +# +# Usage: docker compose -f docker-compose.dev-arm.yaml up -d --build + +services: + db: + image: pgvector/pgvector:pg16 + container_name: postgres + ports: + - ${DB_PORT}:${DB_PORT} + expose: + - ${DB_PORT} + volumes: + - .:/app + - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql + environment: + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_USER=${DB_USER} + - POSTGRES_DB=${DB_NAME} + - PGPORT=${DB_PORT} + command: ["postgres", "-p", "${DB_PORT}"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -p ${DB_PORT}"] + start_period: 0s + start_interval: 0s + interval: 60s + timeout: 5s + retries: 5 + + redis: + image: redis:7.0 + container_name: redis + ports: + - ${REDIS_PORT}:${REDIS_PORT} + expose: + - ${REDIS_PORT} + volumes: + - .:/app + - ./redis.conf:/usr/local/etc/redis/redis.conf + command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--port", "${REDIS_PORT}"] + healthcheck: + test: ["CMD", "redis-cli", "-p", "${REDIS_PORT}", "ping"] + start_period: 0s + start_interval: 0s + interval: 60s + timeout: 5s + retries: 3 + + redis-celery: + image: redis:7.0 + container_name: redis-celery + ports: + - ${CELERY_BROKER_PORT}:${CELERY_BROKER_PORT} + expose: + - ${CELERY_BROKER_PORT} + command: ["redis-server", "--port", "${CELERY_BROKER_PORT}"] + volumes: + - .:/app + healthcheck: + test: ["CMD", "redis-cli", "-p", "${CELERY_BROKER_PORT}", "ping"] + start_period: 0s + start_interval: 0s + interval: 30s + timeout: 5s + retries: 3 + + celery-worker: + build: + context: . + dockerfile: Dockerfile.cpu + container_name: celery-worker + command: [ + "/bin/bash", + "-c", + "source activate myenv && celery -A celery_app.celery_app worker --loglevel=info -E -Q llm,db --concurrency=1 --prefetch-multiplier=1", + ] + depends_on: + redis-celery: + condition: service_healthy + db: + condition: service_healthy + environment: + - REDIS_HOST=redis + - REDIS_PORT=${REDIS_PORT} + - CELERY_BROKER_HOST=redis-celery + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} + - DB_HOST=db + - DB_PORT=${DB_PORT} + - PYTHONPATH=/app/src + - PYTHONMULTIPROCESSING_START_METHOD=spawn + - TOKENIZERS_PARALLELISM=false + - OMP_NUM_THREADS=1 + - HF_TOKEN=${HF_TOKEN} + - HF_HOME=/app/hf_cache + - HF_HUB_CACHE=/app/hf_cache + - TRANSFORMERS_CACHE=/app/hf_cache + - MODEL_CACHE_DIR=/app/hf_cache + - CELERY_WORKER=true + - PRELOAD_MODELS=false + volumes: + - .:/app + - ${DATA_DIR}/hf:/app/hf_cache + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "source activate myenv && celery -A celery_app.celery_app status", + ] + start_period: 0s + interval: 60s + timeout: 10s + retries: 3 + + backend: + build: + context: . + dockerfile: Dockerfile.cpu + container_name: backend + command: ["/bin/bash", "-c", "source activate myenv && python /app/src/main.py"] + expose: + - ${SERVER_PORT} + ports: + - "${SERVER_PORT}:8008" + volumes: + - .:/app + - ${DATA_DIR}/hf:/app/hf_cache + environment: + - SERVER_PORT=${SERVER_PORT} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} + - REDIS_HOST=redis + - REDIS_PORT=${REDIS_PORT} + - CELERY_BROKER_HOST=redis-celery + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} + - DB_HOST=db + - DB_PORT=${DB_PORT} + - WEBSITE_HOST=website + - WEBSITE_PORT=${WEBSITE_PORT} + - PYTHONPATH=/app/src + - HF_TOKEN=${HF_TOKEN} + - HF_HOME=/app/hf_cache + - HF_HUB_CACHE=/app/hf_cache + - TRANSFORMERS_CACHE=/app/hf_cache + - MODEL_CACHE_DIR=/app/hf_cache + - PRELOAD_MODELS=false + # Agent subsystem (ACP) upstream provider config, see .env.example + - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL} + - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY} + - OLLAMA_API_KEY=${OLLAMA_API_KEY} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - GROQ_API_KEY=${GROQ_API_KEY} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} + - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT} + # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an + # Ollama instance running on the host from inside this container. + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + celery-worker: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "http://127.0.0.1:8008/api/ping"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 + + website: + image: node:18 + container_name: website + working_dir: /website + command: bash -c "npm install && npm start -- --port ${WEBSITE_PORT}" + expose: + - ${WEBSITE_PORT} + ports: + - "${WEBSITE_PORT}:${WEBSITE_PORT}" + volumes: + - ./src/website:/website + depends_on: + backend: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "http://localhost:${WEBSITE_PORT}"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 + + nginx: + image: nginx:1.25 + container_name: nginx + ports: + - "8000:8000" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + depends_on: + website: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 9793f34..5eaf26a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,6 +179,7 @@ services: - NVIDIA_DRIVER_CAPABILITIES=compute,utility - NVIDIA_REQUIRE_CUDA=cuda>=12.0 - SERVER_PORT=${SERVER_PORT} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} - REDIS_HOST=redis - REDIS_PORT=${REDIS_PORT} - CELERY_BROKER_HOST=redis-celery @@ -197,6 +198,19 @@ services: - CUDA_VISIBLE_DEVICES=0 - TORCH_CUDA_ARCH_LIST="8.6" - PRELOAD_MODELS=true + # Agent subsystem (ACP) upstream provider config, see .env.example + - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL} + - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY} + - OLLAMA_API_KEY=${OLLAMA_API_KEY} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - GROQ_API_KEY=${GROQ_API_KEY} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} + - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT} + # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an + # Ollama instance running on the host from inside this container (Docker + # Desktop maps this automatically; Linux needs this explicit mapping). + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: db: condition: service_healthy diff --git a/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py b/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py index fdcc32d..e9d14c6 100644 --- a/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py +++ b/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py @@ -5,11 +5,14 @@ mint tasks at all. This seeds one profile per supported runtime so the system is usable out of the box. -Only the built-in ``code4me2-agent`` profile is seeded active, because it is the -one runtime that needs no external binary: it points at a local Ollama endpoint, -which costs nothing and needs no API key. The Goose and Codex profiles are -seeded *inactive* — they require a Goose binary / Node toolchain on the -developer's machine, so an operator flips them on from the admin UI once those +Only the built-in ``code4me2-agent`` profile is seeded active. For initial +testing it points at OpenRouter (an OpenAI-compatible endpoint, per +``agents.provider``) using the free ``cohere/north-mini-code:free`` model, with +``api_key_ref`` set to ``OPENROUTER_API_KEY`` — set that variable in ``.env`` +once a key is available; until then requests are forwarded unauthenticated and +will 401 against OpenRouter. The Goose and Codex profiles are seeded +*inactive* — they require a Goose binary / Node toolchain on the developer's +machine, so an operator flips them on from the admin UI once those prerequisites exist. Inactive profiles are never drawn as A/B arms. No API keys are stored here (or anywhere in the DB): ``api_key_ref`` names the @@ -35,11 +38,12 @@ "profile_id": "7e1f20ce-fc5c-4ba2-b908-eaf77047d4b4", "name": "default-code4me2-agent", "framework_version": "code4me2-agent", - # Ollama speaks the OpenAI-compatible chat-completions format, so the - # generic provider path works against it unchanged. - "base_url": "http://localhost:11434/v1", - "api_key_ref": "OLLAMA_API_KEY", - "model": "qwen2.5-coder:7b", + # OpenRouter speaks the OpenAI-compatible chat-completions format, so + # the generic provider path works against it unchanged. Using the free + # tier for initial testing; swap to a paid model once validated. + "base_url": "https://openrouter.ai/api/v1", + "api_key_ref": "OPENROUTER_API_KEY", + "model": "cohere/north-mini-code:free", "tools_json": ( '["read_file", "write_file", "create_file", "replace_text", ' '"list_files", "search_files", "run_command"]' From deba6052f7bfba225074dbd218a2f4ff60b818b8 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Thu, 10 Sep 2026 11:52:49 +0300 Subject: [PATCH 003/115] Test database service and ignore hygiene --- .dockerignore | 3 + .gitignore | 3 +- docker-compose.dev-arm.yaml | 17 ++++ docker-compose.yml | 33 ++++---- requirements.txt | 4 +- src/database/init_test.sql | 79 ++++++++++++++++++- .../database_tests/test_migration_manager.py | 10 +-- 7 files changed, 120 insertions(+), 29 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6dfbea7..b60f8e2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ venv/ **/__pycache__/ *.pyc +*.egg-info/ # Git .git @@ -13,6 +14,8 @@ node_modules src/website/node_modules # Local caches and data +.env +**/.env .cache .hypothesis .coverage diff --git a/.gitignore b/.gitignore index 21ab2bf..898d4e0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,11 @@ .env node_modules .venv +*.egg-info/ .idea secrets **/__pycache__/ .hypothesis .cache data -.coverage \ No newline at end of file +.coverage diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml index bacd21e..81eee29 100644 --- a/docker-compose.dev-arm.yaml +++ b/docker-compose.dev-arm.yaml @@ -94,6 +94,7 @@ services: - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db - PYTHONPATH=/app/src - PYTHONMULTIPROCESSING_START_METHOD=spawn - TOKENIZERS_PARALLELISM=false @@ -215,3 +216,19 @@ services: interval: 60s timeout: 5s retries: 3 + + test_db: + profiles: ["test"] + image: pgvector/pgvector:pg16 + container_name: postgres_test + ports: + - "5433:5432" + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=test_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d test_db"] + interval: 2s + timeout: 5s + retries: 10 diff --git a/docker-compose.yml b/docker-compose.yml index 5eaf26a..e996ced 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,6 +115,7 @@ services: - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db - PYTHONPATH=/app/src - PYTHONMULTIPROCESSING_START_METHOD=spawn - TOKENIZERS_PARALLELISM=false @@ -270,23 +271,21 @@ services: timeout: 5s retries: 3 -# test_db: -# image: pgvector/pgvector:pg16 -# container_name: postgres_test -# ports: -# - "5433:5432" -# environment: -# - POSTGRES_PASSWORD=postgres -# - POSTGRES_USER=postgres -# - POSTGRES_DB=test_db -# volumes: -# - .:/app -# - ./src/database/init_test.sql:/docker-entrypoint-initdb.d/init_test.sql -# healthcheck: -# test: ["CMD-SHELL", "pg_isready -U postgres"] -# interval: 5s -# timeout: 5s -# retries: 5 + test_db: + profiles: ["test"] + image: pgvector/pgvector:pg16 + container_name: postgres_test + ports: + - "5433:5432" + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=test_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d test_db"] + interval: 2s + timeout: 5s + retries: 10 volumes: postgres_data: diff --git a/requirements.txt b/requirements.txt index b62198b..60d6ec5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,9 @@ python-multipart~=0.0.20 # Database # ----------------------- SQLAlchemy~=2.0.30 -psycopg_binary~=3.2.1 +# Install psycopg and its binary implementation as one version-locked extra. +# Separate requirements allow pip to select incompatible minor versions. +psycopg[binary]~=3.2.1 psycopg2-binary~=2.9.9 alembic>=1.13.1 diff --git a/src/database/init_test.sql b/src/database/init_test.sql index 4c93cc7..e26ddbd 100644 --- a/src/database/init_test.sql +++ b/src/database/init_test.sql @@ -22,7 +22,8 @@ CREATE TABLE IF NOT EXISTS public."user" verified BOOLEAN DEFAULT FALSE, config_id BIGINT NOT NULL, preference TEXT, - auth_token uuid NULL + auth_token uuid NULL, + is_admin BOOLEAN DEFAULT FALSE ); -- Model name table with instruction tuning flag @@ -198,6 +199,30 @@ CREATE TABLE IF NOT EXISTS public.documentation created_at timestamp with time zone NOT NULL DEFAULT NOW() ); +-- Study table for A/B testing and user experiments +CREATE TABLE IF NOT EXISTS public.study +( + study_id uuid NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + created_by uuid NOT NULL, + starts_at timestamp with time zone NOT NULL, + ends_at timestamp with time zone, + is_active BOOLEAN DEFAULT FALSE, + default_config_id BIGINT NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() +); + +-- Config assignment history for tracking user treatments +CREATE TABLE IF NOT EXISTS public.config_assignment_history +( + user_id uuid NOT NULL, + study_id uuid NOT NULL, + assigned_config_id BIGINT NOT NULL, + assigned_at timestamp with time zone NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, study_id) +); + -- Indexes for documentation table CREATE INDEX IF NOT EXISTS idx_documentation_language ON public.documentation (language); CREATE INDEX IF NOT EXISTS idx_documentation_embedding ON public.documentation @@ -341,9 +366,41 @@ ALTER TABLE public.session_projects REFERENCES public.project(project_id) ON DELETE CASCADE; +-- Foreign keys for study management +ALTER TABLE public.study + ADD CONSTRAINT fk_study_created_by FOREIGN KEY (created_by) + REFERENCES public."user" (user_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.study + ADD CONSTRAINT fk_study_default_config FOREIGN KEY (default_config_id) + REFERENCES public.config (config_id) + ON UPDATE NO ACTION + ON DELETE NO ACTION; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_user FOREIGN KEY (user_id) + REFERENCES public."user" (user_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_study FOREIGN KEY (study_id) + REFERENCES public.study (study_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_config FOREIGN KEY (assigned_config_id) + REFERENCES public.config (config_id) + ON UPDATE NO ACTION + ON DELETE NO ACTION; + -- Indexes for performance CREATE INDEX IF NOT EXISTS idx_user_email ON public."user" (email); CREATE INDEX IF NOT EXISTS idx_user_config_id ON public."user" (config_id); +CREATE INDEX IF NOT EXISTS idx_user_is_admin ON public."user" (is_admin); CREATE INDEX IF NOT EXISTS idx_ctxt_telemetry_version_id ON public.contextual_telemetry (version_id); CREATE INDEX IF NOT EXISTS idx_ctxt_telemetry_trigger_type_id ON public.contextual_telemetry (trigger_type_id); @@ -359,17 +416,33 @@ CREATE INDEX IF NOT EXISTS idx_session_projects_project_id ON public.session_pro CREATE INDEX IF NOT EXISTS idx_chat_project_id ON public.chat (project_id); CREATE INDEX IF NOT EXISTS idx_chat_user_id ON public.chat (user_id); +-- Analytics-optimized indexes CREATE INDEX IF NOT EXISTS idx_meta_query_user_id ON public.meta_query (user_id); CREATE INDEX IF NOT EXISTS idx_meta_query_project_id ON public.meta_query (project_id); CREATE INDEX IF NOT EXISTS idx_meta_query_session_id ON public.meta_query (session_id); CREATE INDEX IF NOT EXISTS idx_meta_query_type ON public.meta_query (query_type); +CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp ON public.meta_query ("timestamp"); +CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp_type ON public.meta_query ("timestamp", query_type); CREATE INDEX IF NOT EXISTS idx_chat_query_chat_id ON public.chat_query (chat_id); CREATE INDEX IF NOT EXISTS idx_had_generation_meta_query_model ON public.had_generation (meta_query_id, model_id); +CREATE INDEX IF NOT EXISTS idx_had_generation_model_id ON public.had_generation (model_id); +CREATE INDEX IF NOT EXISTS idx_had_generation_was_accepted ON public.had_generation (was_accepted); +CREATE INDEX IF NOT EXISTS idx_had_generation_confidence ON public.had_generation (confidence); CREATE INDEX IF NOT EXISTS idx_ground_truth_completion_query_timestamp ON public.ground_truth (completion_query_id, truth_timestamp); +-- Study management indexes +CREATE INDEX IF NOT EXISTS idx_study_is_active ON public.study (is_active); +CREATE INDEX IF NOT EXISTS idx_study_created_by ON public.study (created_by); +CREATE INDEX IF NOT EXISTS idx_study_starts_at ON public.study (starts_at); +CREATE INDEX IF NOT EXISTS idx_study_ends_at ON public.study (ends_at); + +CREATE INDEX IF NOT EXISTS idx_config_assignment_user_id ON public.config_assignment_history (user_id); +CREATE INDEX IF NOT EXISTS idx_config_assignment_study_id ON public.config_assignment_history (study_id); +CREATE INDEX IF NOT EXISTS idx_config_assignment_assigned_at ON public.config_assignment_history (assigned_at); + -- Insert default data INSERT INTO public.config (config_data) VALUES ('config { // module configuration @@ -608,7 +681,7 @@ INSERT INTO public.config (config_data) VALUES ('config { // Authentication Settings auth { google { - clientId = "67736337656-un249ihklv5i93n033i1v46q12bfv5g2.apps.googleusercontent.com" + clientId = "288822392430-se6cmstnkje31e5kv32u95lo8544ltu4.apps.googleusercontent.com" } } // model configuration @@ -857,4 +930,4 @@ INSERT INTO public.plugin_version (version_name, ide_type, description) VALUES ('0.1.0j', 'JetBrains', 'enhanced version with chat support'), ('0.1.0v', 'VSCode', 'enhanced version with chat support'); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/tests/database_tests/test_migration_manager.py b/tests/database_tests/test_migration_manager.py index 540eb73..e36a58e 100644 --- a/tests/database_tests/test_migration_manager.py +++ b/tests/database_tests/test_migration_manager.py @@ -9,13 +9,9 @@ src_dir = project_root / "src" sys.path.insert(0, str(src_dir)) -# Mock the alembic imports at module level -sys.modules["alembic"] = MagicMock() -sys.modules["alembic.command"] = MagicMock() -sys.modules["alembic.config"] = MagicMock() -sys.modules["sqlalchemy"] = MagicMock() - -# Now import after mocking +# Import the real module, then patch its collaborators within individual tests. +# Replacing sqlalchemy or alembic in sys.modules here corrupts every database +# test collected in the same pytest process. from database.migration.migration_manager import MigrationManager From d35dce3d94228a9200037d5f5321b387fcbf4bd7 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Thu, 10 Sep 2026 11:52:53 +0300 Subject: [PATCH 004/115] Migrate Python agent to agent-client-protocol 0.12 on stable v1 --- pyproject.toml | 8 +- src/code4me2_agent/acp_runtime.py | 307 ++++++++++++++++------- src/code4me2_agent/acp_updates.py | 17 +- src/code4me2_agent/adapters.py | 132 +++++++++- src/code4me2_agent/echo.py | 14 +- src/code4me2_agent/mcp_tools.py | 259 +++++++++++++++++++ tests/code4me2_agent/__init__.py | 1 + tests/code4me2_agent/mcp_toy_server.py | 23 ++ tests/code4me2_agent/test_acp_runtime.py | 260 +++++++++++++++++++ 9 files changed, 916 insertions(+), 105 deletions(-) create mode 100644 src/code4me2_agent/mcp_tools.py create mode 100644 tests/code4me2_agent/__init__.py create mode 100644 tests/code4me2_agent/mcp_toy_server.py create mode 100644 tests/code4me2_agent/test_acp_runtime.py diff --git a/pyproject.toml b/pyproject.toml index 00bac41..d61d585 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,11 @@ version = "0.1.0" description = "Code4Me ACP agent runtime" requires-python = ">=3.10" dependencies = [ - "agent-client-protocol>=0.8", + # Pre-1.0 SDK: allow bugfix releases in the 0.12 series. Compatibility is + # covered by the ACP runtime tests; minor bumps need a re-run, not a migration. + "agent-client-protocol~=0.12.1", + # ACP requires agents to support client-supplied stdio MCP servers. + "mcp==2.2.0", "openai>=1.0,<2", ] @@ -24,4 +28,4 @@ line-length = 99 [tool.ruff.lint] select = ["E", "F", "W", "Q", "I", "TCH"] -ignore = ["E203", "E402", "E501", "F841", "F401", "F811"] \ No newline at end of file +ignore = ["E203", "E402", "E501", "F841", "F401", "F811"] diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py index 0283168..55571c1 100644 --- a/src/code4me2_agent/acp_runtime.py +++ b/src/code4me2_agent/acp_runtime.py @@ -2,8 +2,9 @@ import asyncio import logging -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path +from threading import Event from typing import TYPE_CHECKING, Any from uuid import uuid4 @@ -13,10 +14,20 @@ from code4me2_agent.command_tools import build_acp_command_backend from code4me2_agent.echo import EchoAgentCore from code4me2_agent.file_tools import build_acp_file_system_backend +from code4me2_agent.mcp_tools import StdioMcpToolBroker, serialize_mcp_servers from code4me2_agent.runtime_auth import AcpAuthorizationFailure, AcpBackendAuthorization logger = logging.getLogger(__name__) +# The runtime currently implements ACP protocol version 1. +SUPPORTED_PROTOCOL_VERSIONS: tuple[int, ...] = (1,) + +# Python SDK 0.12.1 still guards the now-stable session/resume and session/close +# routes with this SDK-wide switch. Unsupported guarded routes are safe because +# the concrete agent below deliberately does not inherit the SDK Protocol stubs. +_ENABLE_STABLE_SESSION_ROUTE_WORKAROUND = True + + if TYPE_CHECKING: from acp.interfaces import Client @@ -112,6 +123,10 @@ def _acp_stop_reason(adapter_stop_reason: str) -> str: class AgentSession: session_id: str core: EchoAgentCore + mcp_tools: StdioMcpToolBroker | None = None + cancel_event: Event = field(default_factory=Event) + prompt_lock: asyncio.Lock = field(default_factory=asyncio.Lock) + active_prompt_task: asyncio.Task[Any] | None = None class AcpSessionEventSink: @@ -279,6 +294,19 @@ def _persisted_memory_messages(payload: dict) -> list[dict[str, Any]]: return [message for message in messages if isinstance(message, dict)] +def _is_supported_stdio_mcp_server(server: object) -> bool: + """Check the 0.12.1 stdio shape (name/command/args/env) without spawning. + + HTTP, SSE and malformed entries are rejected as unsupported before the + broker ever runs: only validated stdio reaches process startup, whose own + failures (missing binary, timeout, duplicates) stay mcp_server_start_failed. + """ + for key in ("name", "command", "args", "env"): + if capability_value(server, key) is None: + return False + return True + + def _should_record_runtime_context( *, client_capabilities: object | None, @@ -287,26 +315,29 @@ def _should_record_runtime_context( return client_capabilities is not None or session_lookup != "existing" -def create_acp_agent(config: AgentConfig, *, authorization: Any | None = None) -> Any: +def create_acp_agent( + config: AgentConfig, + *, + authorization: Any | None = None, +) -> Any: # TODO new agent -> a lot of changes to here. try: from acp import ( - Agent, AuthenticateResponse, InitializeResponse, - LoadSessionResponse, NewSessionResponse, PromptResponse, RequestError, ) from acp.schema import ( AgentCapabilities, - AuthMethod, + AuthMethodAgent, Implementation, McpCapabilities, PromptCapabilities, ResumeSessionResponse, SessionCapabilities, + SessionCloseCapabilities, SessionResumeCapabilities, ) except ImportError as exc: @@ -315,13 +346,12 @@ def create_acp_agent(config: AgentConfig, *, authorization: Any | None = None) - "Install this package from source with agent-client-protocol available." ) from exc - class Code4MeEchoAgent(Agent): + class Code4MeEchoAgent: _conn: Client _client_capabilities: object | None _updates: AcpUpdateBuilder _bootstrap_core: EchoAgentCore _sessions: dict[str, AgentSession] - _cancelled_sessions: set[str] _authorization: Any _authorized_workspace: Path | None @@ -330,7 +360,6 @@ def on_connect(self, conn: Client) -> None: self._client_capabilities = None self._updates = AcpUpdateBuilder() self._sessions = {} - self._cancelled_sessions = set() self._authorization = ( authorization or AcpBackendAuthorization.from_environment( @@ -344,7 +373,11 @@ def on_connect(self, conn: Client) -> None: self.command_tools = self._bootstrap_core.command_tools def _build_session( - self, *, session_config: AgentConfig, session_id: str + self, + *, + session_config: AgentConfig, + session_id: str, + mcp_tools: StdioMcpToolBroker | None, ) -> AgentSession: async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) event_sink = AcpSessionEventSink( @@ -369,8 +402,13 @@ def _build_session( client_capabilities=self._client_capabilities, async_runner=async_runner, ), + mcp_tools=mcp_tools, + ) + return AgentSession( + session_id=session_id, + core=core, + mcp_tools=mcp_tools, ) - return AgentSession(session_id=session_id, core=core) def _load_persisted_memory(self, session: AgentSession) -> None: request_json = getattr(self._authorization, "authorized_json_request", None) @@ -446,12 +484,13 @@ def _session_config( ), ) - async def _create_or_load_session( + async def _create_or_resume_session( self, *, cwd: str, session_id: str, event_type: str, + mcp_servers: list[Any] | None, ) -> AgentSession: await self._require_authenticated() workspace_root = _resolve_session_cwd(cwd) @@ -461,10 +500,37 @@ async def _create_or_load_session( workspace_root=workspace_root, session_id=session_id, ) - session = self._build_session( - session_config=session_config, session_id=session_id - ) + try: + mcp_tools = await asyncio.to_thread( + StdioMcpToolBroker.open, + list(mcp_servers or []), + cwd=workspace_root, + ) + except (RuntimeError, TimeoutError, ValueError) as exc: + # Best effort: a failing tool sidecar (e.g. the IDE-bundled MCP + # server binary refusing a second instance) must not take down + # the whole chat. The session keeps its native file/terminal + # tools; malformed entries are still rejected up front. + logger.warning( + "ACP session continuing without MCP tools after broker failure %s: %s", + serialize_mcp_servers(mcp_servers), + exc, + ) + mcp_tools = None + try: + session = self._build_session( + session_config=session_config, + session_id=session_id, + mcp_tools=mcp_tools, + ) + except BaseException: + if mcp_tools is not None: + await asyncio.to_thread(mcp_tools.close) + raise await asyncio.to_thread(self._load_persisted_memory, session) + previous_session = self._sessions.get(session_id) + if previous_session is not None: + await self._stop_session(previous_session) self._sessions[session_id] = session self.file_tools = session.core.file_tools self.command_tools = session.core.command_tools @@ -481,17 +547,45 @@ async def _create_or_load_session( "client_capabilities": _normalize_capabilities( self._client_capabilities ), + "mcp_servers_requested": serialize_mcp_servers(mcp_servers), + "mcp_backend": "broker" + if session.mcp_tools is not None + else "none", **_prompt_backend_state(session), }, ) return session + @staticmethod + async def _stop_active_prompt(session: AgentSession) -> None: + session.cancel_event.set() + active_task = session.active_prompt_task + if active_task is None or active_task is asyncio.current_task(): + return + try: + await asyncio.shield(active_task) + except Exception: # noqa: BLE001 + logger.exception( + "Active ACP prompt failed while the session was being replaced or closed." + ) + + @classmethod + async def _stop_session(cls, session: AgentSession) -> None: + await cls._stop_active_prompt(session) + if session.mcp_tools is not None: + await asyncio.to_thread(session.mcp_tools.close) + async def initialize( self, protocol_version: int, client_capabilities: object | None = None, **kwargs: Any, ) -> InitializeResponse: + negotiated_protocol_version = ( + protocol_version + if protocol_version in SUPPORTED_PROTOCOL_VERSIONS + else max(SUPPORTED_PROTOCOL_VERSIONS) + ) self._client_capabilities = client_capabilities or kwargs.get( "clientCapabilities" ) @@ -548,9 +642,11 @@ async def initialize( }, ) return InitializeResponse( - protocol_version=protocol_version, + protocol_version=negotiated_protocol_version, agent_capabilities=AgentCapabilities( - loadSession=True, + # This agent restores model memory, but does not yet persist the + # client-visible update stream required by session/load replay. + loadSession=False, promptCapabilities=PromptCapabilities( image=False, audio=False, @@ -558,6 +654,7 @@ async def initialize( ), sessionCapabilities=SessionCapabilities( resume=SessionResumeCapabilities(), + close=SessionCloseCapabilities(), ), mcpCapabilities=McpCapabilities( http=False, @@ -570,7 +667,7 @@ async def initialize( version="0.1.0", ), auth_methods=[ - AuthMethod( + AuthMethodAgent( id="code4me-plugin-session", name="Code4Me Plugin Session", description="Authenticate using a prepared Code4Me project session.", @@ -716,51 +813,84 @@ async def _require_authenticated(self) -> None: async def new_session( self, cwd: str, + mcp_servers: list[Any] | None = None, + additional_directories: list[str] | None = None, **kwargs: Any, ) -> NewSessionResponse: + self._reject_unsupported_session_inputs( + mcp_servers=mcp_servers, + additional_directories=additional_directories, + ) session_id = str( kwargs.get("session_id") or kwargs.get("sessionId") or uuid4().hex ) - await self._create_or_load_session( + await self._create_or_resume_session( cwd=cwd, session_id=session_id, event_type="agent.acp.session_created", + mcp_servers=mcp_servers, ) return NewSessionResponse(session_id=session_id) - async def load_session( - self, - cwd: str, - session_id: str, - **kwargs: Any, - ) -> LoadSessionResponse: - await self._create_or_load_session( - cwd=cwd, - session_id=session_id, - event_type="agent.acp.session_loaded", - ) - return LoadSessionResponse() - async def resume_session( self, cwd: str, session_id: str, + mcp_servers: list[Any] | None = None, + additional_directories: list[str] | None = None, **kwargs: Any, ) -> ResumeSessionResponse: - await self._create_or_load_session( + self._reject_unsupported_session_inputs( + mcp_servers=mcp_servers, + additional_directories=additional_directories, + ) + await self._create_or_resume_session( cwd=cwd, session_id=session_id, event_type="agent.acp.session_resumed", + mcp_servers=mcp_servers, ) return ResumeSessionResponse() + @staticmethod + def _reject_unsupported_session_inputs( + *, + mcp_servers: list[Any] | None, + additional_directories: list[str] | None, + ) -> None: + if additional_directories: + raise RequestError.invalid_params( + {"reason": "additional_directories_not_supported"} + ) + for server in mcp_servers or []: + if not _is_supported_stdio_mcp_server(server): + raise RequestError.invalid_params( + {"reason": "mcp_servers_not_supported"} + ) + + async def close_session( + self, + session_id: str, + **kwargs: Any, + ) -> None: + await self._require_authenticated() + session = self._sessions.get(session_id) + if session is None: + raise RequestError.invalid_params({"reason": "unknown_session"}) + try: + await self._stop_session(session) + finally: + self._sessions.pop(session_id, None) + async def cancel( self, session_id: str, **kwargs: Any, ) -> None: await self._require_authenticated() - self._cancelled_sessions.add(session_id) + session = self._sessions.get(session_id) + if session is not None and session.active_prompt_task is not None: + session.cancel_event.set() async def prompt( self, @@ -778,61 +908,63 @@ async def prompt( {"reason": "authorization_rejected"} ) from None request_id = message_id or uuid4().hex - if session_id in self._cancelled_sessions: - self._cancelled_sessions.discard(session_id) - return PromptResponse( - stop_reason="cancelled", - user_message_id=message_id, - ) session = self._sessions.get(session_id) if session is None: raise RequestError.auth_required({"reason": "unknown_session"}) - if _should_record_runtime_context( - client_capabilities=self._client_capabilities, - session_lookup="existing", - ): - _record_acp_runtime_event( - session.core._telemetry, - event_type="agent.acp.prompt_context", - session_id=session_id, - request_id=request_id, - payload={ - "session_lookup": "existing", - "known_session_ids": sorted(self._sessions.keys()), - "client_capabilities": _normalize_capabilities( - self._client_capabilities - ), - **_prompt_backend_state(session), - }, - ) - prompt_text = await _prompt_text_async(prompt) - # 1 - logger.info("Prompt text: %s", prompt_text) - result = await asyncio.to_thread( - session.core.handle_prompt, - prompt_text, - request_id, - message_id, - ) - await asyncio.to_thread(self._save_persisted_memory, session) - # n - await self._conn.session_update( - session_id=session_id, - update=self._updates.agent_message( - result.final_response, - metadata={ - "code4me2": { - "phase": "completed", - "durationMs": result.duration_ms, - } - }, - ), - source="code4me2_agent", - ) - return PromptResponse( - stop_reason=_acp_stop_reason(result.stop_reason), - user_message_id=message_id, - ) + async with session.prompt_lock: + session.cancel_event.clear() + session.active_prompt_task = asyncio.current_task() + try: + if _should_record_runtime_context( + client_capabilities=self._client_capabilities, + session_lookup="existing", + ): + _record_acp_runtime_event( + session.core._telemetry, + event_type="agent.acp.prompt_context", + session_id=session_id, + request_id=request_id, + payload={ + "session_lookup": "existing", + "known_session_ids": sorted(self._sessions.keys()), + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + **_prompt_backend_state(session), + }, + ) + prompt_text = await _prompt_text_async(prompt) + logger.info("Prompt text: %s", prompt_text) + result = await asyncio.to_thread( + session.core.handle_prompt, + prompt_text, + request_id, + message_id, + None, + session.cancel_event, + ) + await asyncio.to_thread(self._save_persisted_memory, session) + if result.final_response: + await self._conn.session_update( + session_id=session_id, + update=self._updates.agent_message( + result.final_response, + message_id=request_id, + metadata={ + "code4me2": { + "phase": "completed", + "durationMs": result.duration_ms, + } + }, + ), + source="code4me2_agent", + ) + return PromptResponse( + stop_reason=_acp_stop_reason(result.stop_reason), + ) + finally: + session.active_prompt_task = None + session.cancel_event.clear() return Code4MeEchoAgent() @@ -853,4 +985,7 @@ async def run_acp_stdio(config: AgentConfig) -> None: "Install this package from source with agent-client-protocol available." ) from exc - await run_agent(create_acp_agent(config), use_unstable_protocol=True) + await run_agent( + create_acp_agent(config), + use_unstable_protocol=_ENABLE_STABLE_SESSION_ROUTE_WORKAROUND, + ) diff --git a/src/code4me2_agent/acp_updates.py b/src/code4me2_agent/acp_updates.py index 152314d..8488e79 100644 --- a/src/code4me2_agent/acp_updates.py +++ b/src/code4me2_agent/acp_updates.py @@ -37,7 +37,13 @@ def helper_gaps(self) -> tuple[str, ...]: ) return tuple(name for name in required_helpers if not hasattr(self._acp, name)) - def agent_message(self, text: str, *, metadata: dict[str, Any] | None = None) -> Any: + def agent_message( + self, + text: str, + *, + message_id: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> Any: text_block = getattr(self._acp, "text_block", None) update_agent_message = getattr(self._acp, "update_agent_message", None) if text_block is not None and update_agent_message is not None: @@ -47,9 +53,14 @@ def agent_message(self, text: str, *, metadata: dict[str, Any] | None = None) -> session_update="agent_message_chunk", content=self._schema.TextContentBlock(type="text", text=text), ) - if metadata is None: + model_updates: dict[str, Any] = {} + if message_id is not None: + model_updates["message_id"] = message_id + if metadata is not None: + model_updates["field_meta"] = metadata + if not model_updates: return update - return update.model_copy(update={"field_meta": metadata}) + return update.model_copy(update=model_updates) def agent_thought(self, text: str, *, metadata: dict[str, Any] | None = None) -> Any: text_block = getattr(self._acp, "text_block", None) diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py index 42f016d..1ee17a3 100644 --- a/src/code4me2_agent/adapters.py +++ b/src/code4me2_agent/adapters.py @@ -10,6 +10,7 @@ import logging import fcntl from pathlib import Path +from threading import Event from time import time if TYPE_CHECKING: @@ -17,6 +18,7 @@ from code4me2_agent.config import AgentConfig from code4me2_agent.events import AgentEventSink from code4me2_agent.file_tools import WorkspaceFileTools + from code4me2_agent.mcp_tools import StdioMcpToolBroker from code4me2_agent.telemetry import AgentTelemetryRecorder @@ -37,6 +39,7 @@ def handle_prompt( request_id: str, message_id: str | None, memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, ) -> AdapterResult: ... @@ -72,6 +75,21 @@ class BackendProviderRequestError(RuntimeError): pass +def _turn_was_cancelled(cancellation_event: Event | None) -> bool: + return cancellation_event is not None and cancellation_event.is_set() + + +def _cancelled_result( + thoughts: list[str] | tuple[str, ...] = (), +) -> AdapterResult: + return AdapterResult( + final_response="", + stop_reason="cancelled", + run_status="cancelled", + thoughts=tuple(thoughts), + ) + + def _rate_limit_provider_request_from_env() -> None: state_path_text = os.getenv("CODE4ME_RATE_LIMIT_STATE_PATH", "").strip() if not state_path_text: @@ -195,7 +213,10 @@ def handle_prompt( request_id: str, message_id: str | None, memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, ) -> AdapterResult: + if _turn_was_cancelled(cancellation_event): + return _cancelled_result() return AdapterResult( final_response=f"Code4Me ACP echo: {prompt}", stop_reason="end_turn", @@ -210,6 +231,7 @@ def __init__( command_tools: WorkspaceCommandTools, *, event_sink: AgentEventSink | None = None, + mcp_tools: StdioMcpToolBroker | None = None, ) -> None: from code4me2_agent.events import ( NoopAgentEventSink, @@ -219,6 +241,7 @@ def __init__( self._file_tools = file_tools self._command_tools = command_tools + self._mcp_tools = mcp_tools self._event_sink = event_sink or NoopAgentEventSink() self._event_type = ToolCallEvent self._thought_event_type = ThoughtEvent @@ -289,6 +312,8 @@ def execute( run_id=run_id, request_id=request_id, ) + elif self._mcp_tools is not None and self._mcp_tools.has_tool(name): + result = self._mcp_tools.execute(name, arguments) else: raise ToolRegistryError( f"Unsupported tool name: {name}", @@ -305,6 +330,22 @@ def execute( ) return tool_output + def definitions(self) -> list[dict[str, Any]]: + definitions = _tool_definitions() + if self._mcp_tools is not None: + definitions.extend(self._mcp_tools.definitions()) + return definitions + + def known_tool_names(self) -> set[str]: + names: set[str] = set() + for tool in self.definitions(): + function = tool.get("function") + if isinstance(function, dict): + name = str(function.get("name", "")).strip() + if name: + names.add(name) + return names + def _emit_tool_start( self, tool_call: ToolCall, @@ -412,6 +453,7 @@ def __init__( api_key_env: str, timeout_seconds: float, auth_headers: dict[str, str] | None = None, + tool_definitions: list[dict[str, Any]] | None = None, ) -> None: self._kind = kind.strip() or "code4me_backend" self._base_url = base_url.rstrip("/") @@ -419,12 +461,13 @@ def __init__( self._api_key_env = api_key_env self._timeout_seconds = timeout_seconds self._auth_headers = dict(auth_headers or {}) + self._tool_definitions = list(tool_definitions or _tool_definitions()) def generate(self, messages: list[dict[str, Any]]) -> ProviderTurn: request_payload = { "model": self._model, "messages": [_to_openai_message(message) for message in messages], - "tools": _tool_definitions(), + "tools": self._tool_definitions, "tool_choice": "auto", } @@ -589,6 +632,7 @@ def handle_prompt( request_id: str, message_id: str | None, memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, ) -> AdapterResult: provider = self._provider() if memory is None: @@ -601,6 +645,8 @@ def handle_prompt( memory.append({"role": "user", "content": prompt}) thoughts: list[str] = [] for iteration in range(1, self._config.adapter.max_iterations + 1): + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) messages = memory.window() thought_started_at = perf_counter() self._event_sink.thought( @@ -684,6 +730,13 @@ def handle_prompt( f"The provider request failed: {exc}", stop_reason="error", ) + if _turn_was_cancelled(cancellation_event): + self._complete_thought_phase( + run_id=run_id, + request_id=request_id, + started_at=thought_started_at, + ) + return _cancelled_result(thoughts) # 5 parsed = self._parse_output(output, run_id=run_id, request_id=request_id) if parsed is None: @@ -712,6 +765,8 @@ def handle_prompt( ) ) for tool_call in parsed.tool_calls: + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) self._telemetry.record( event_type="agent.tool.called", run_id=run_id, @@ -783,6 +838,8 @@ def handle_prompt( error_message=str(exc), thoughts=thoughts, ) + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) memory.append( { "role": "tool", @@ -793,6 +850,8 @@ def handle_prompt( ) continue if parsed.final_answer is not None: + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) if ( iteration < self._config.adapter.max_iterations and _USER_REQUESTED_FILE_CHANGE_RE.search(prompt) @@ -809,11 +868,15 @@ def handle_prompt( arguments={"path": file_path, "content": file_content}, ) try: + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) tool_output = self._tool_registry.execute( tool_call, run_id=run_id, request_id=request_id, ) + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) except ToolRegistryError: pass except PermissionError: @@ -866,6 +929,8 @@ def handle_prompt( ) continue + if _turn_was_cancelled(cancellation_event): + return _cancelled_result(thoughts) self._record_loop_failure( run_id=run_id, request_id=request_id, @@ -944,6 +1009,7 @@ def _provider(self) -> FakeOpenAICompatibleProvider | OpenAICompatibleProvider: api_key_env=self._config.adapter.provider.api_key_env, timeout_seconds=self._config.adapter.provider.timeout_seconds, auth_headers=self._config.adapter.provider.auth_headers, + tool_definitions=self._tool_registry.definitions(), ) def _system_context(self) -> str: @@ -963,6 +1029,8 @@ def _system_context(self) -> str: "When calling tools, prefer workspace-relative paths and cwd='.' unless the user explicitly asks for an absolute path. " "Do not invent container paths such as /workspace. " "After file tools complete, answer directly in assistant text; do not run commands just to print a confirmation." + " Tools whose names start with mcp__ come from MCP servers configured by the ACP client; " + "use them when their descriptions match the user's request." ) def _assistant_tool_call_message( @@ -1016,7 +1084,10 @@ def _parse_output( final_answer = raw_output.get("final_answer") if isinstance(tool_calls_data, list) or isinstance(final_answer, str): # 6 - TODO here might be the problem in tool calling returns - fallback_tool_call = _single_json_tool_call_from_text(final_answer) + fallback_tool_call = _single_json_tool_call_from_text( + final_answer, + known_tool_names=self._tool_registry.known_tool_names(), + ) if fallback_tool_call is not None and not tool_calls_data: return ParsedProviderOutput( tool_calls=[fallback_tool_call], @@ -1064,7 +1135,10 @@ def _parse_output( failure_reason="json_text_not_object", ) return None - fallback_tool_call = _single_tool_call_from_object(parsed_text) + fallback_tool_call = _single_tool_call_from_object( + parsed_text, + known_tool_names=self._tool_registry.known_tool_names(), + ) if fallback_tool_call is not None: return ParsedProviderOutput( tool_calls=[fallback_tool_call], @@ -1182,13 +1256,17 @@ def create_agent_adapter( file_tools: WorkspaceFileTools, command_tools: WorkspaceCommandTools, event_sink: AgentEventSink | None = None, + mcp_tools: StdioMcpToolBroker | None = None, ) -> AgentAdapter: if config.adapter.name == "openai_compatible_react": return OpenAICompatibleReactAdapter( config, telemetry=telemetry, tool_registry=ToolRegistry( - file_tools, command_tools, event_sink=event_sink + file_tools, + command_tools, + event_sink=event_sink, + mcp_tools=mcp_tools, ), event_sink=event_sink, ) @@ -1231,6 +1309,15 @@ def _tool_event_metadata( fallback_title="Replace text", content_prefix="Replaced text in", ) + if tool_name.startswith("mcp__"): + title = f"Call MCP tool {tool_name.removeprefix('mcp__')}" + return { + "kind": "other", + "title": title, + "content_text": title, + "raw_input": values, + "raw_output": values, + } return None @@ -1273,15 +1360,22 @@ def _normalize_tool_calls(value: Any) -> list[ToolCall]: # TODO check here -def _single_json_tool_call_from_text(value: Any) -> ToolCall | None: +def _single_json_tool_call_from_text( + value: Any, + *, + known_tool_names: set[str] | None = None, +) -> ToolCall | None: if not isinstance(value, str) or not value.strip(): return None text = value.strip() - result = _try_parse_json(text) + result = _try_parse_json(text, known_tool_names=known_tool_names) if result is not None: return result for match in _CODE_FENCE_RE.finditer(text): - result = _try_parse_json(match.group(1).strip()) + result = _try_parse_json( + match.group(1).strip(), + known_tool_names=known_tool_names, + ) if result is not None: return result idx = text.find('{"name":') @@ -1293,7 +1387,10 @@ def _single_json_tool_call_from_text(value: Any) -> ToolCall | None: elif text[i] == "}": brace_count -= 1 if brace_count == 0: - result = _try_parse_json(text[idx : i + 1]) + result = _try_parse_json( + text[idx : i + 1], + known_tool_names=known_tool_names, + ) if result is not None: return result return None @@ -1343,7 +1440,11 @@ def _extract_content_from_code_block(text: str) -> str | None: return match.group(1).strip() -def _try_parse_json(text: str) -> ToolCall | None: +def _try_parse_json( + text: str, + *, + known_tool_names: set[str] | None = None, +) -> ToolCall | None: if not text: return None try: @@ -1352,12 +1453,19 @@ def _try_parse_json(text: str) -> ToolCall | None: return None if not isinstance(parsed, dict): return None - return _single_tool_call_from_object(parsed) + return _single_tool_call_from_object( + parsed, + known_tool_names=known_tool_names, + ) -def _single_tool_call_from_object(value: dict[str, Any]) -> ToolCall | None: +def _single_tool_call_from_object( + value: dict[str, Any], + *, + known_tool_names: set[str] | None = None, +) -> ToolCall | None: name = str(value.get("name", "")).strip() - if name not in _known_tool_names(): + if name not in (known_tool_names or _known_tool_names()): return None arguments = value.get("arguments", {}) if not isinstance(arguments, dict): diff --git a/src/code4me2_agent/echo.py b/src/code4me2_agent/echo.py index d44981e..7998201 100644 --- a/src/code4me2_agent/echo.py +++ b/src/code4me2_agent/echo.py @@ -3,6 +3,7 @@ import logging from dataclasses import dataclass from time import perf_counter +from threading import Event from typing import TYPE_CHECKING from uuid import uuid4 @@ -16,6 +17,7 @@ if TYPE_CHECKING: from code4me2_agent.config import AgentConfig from code4me2_agent.events import AgentEventSink + from code4me2_agent.mcp_tools import StdioMcpToolBroker @dataclass(frozen=True) @@ -51,6 +53,7 @@ def rebuild_tools( *, acp_file_backend: object | None = None, acp_command_backend: object | None = None, + mcp_tools: StdioMcpToolBroker | None = None, ) -> None: self.file_tools = WorkspaceFileTools( self._config, @@ -62,15 +65,20 @@ def rebuild_tools( acp_backend=acp_command_backend, telemetry=self._telemetry, ) - self.rebuild_adapter() + self.rebuild_adapter(mcp_tools=mcp_tools) - def rebuild_adapter(self) -> None: + def rebuild_adapter( + self, + *, + mcp_tools: StdioMcpToolBroker | None = None, + ) -> None: self._adapter = create_agent_adapter( self._config, telemetry=self._telemetry, file_tools=self.file_tools, command_tools=self.command_tools, event_sink=self._event_sink, + mcp_tools=mcp_tools, ) def _build_session_memory(self) -> MemoryWindow | None: @@ -99,6 +107,7 @@ def handle_prompt( request_id: str | None = None, message_id: str | None = None, run_id: str | None = None, + cancellation_event: Event | None = None, ) -> EchoPromptResult: started_at = perf_counter() request_id = request_id or uuid4().hex @@ -129,6 +138,7 @@ def handle_prompt( request_id=request_id, message_id=message_id, memory=self._session_memory, + cancellation_event=cancellation_event, ) # n-1 final_response = adapter_result.final_response diff --git a/src/code4me2_agent/mcp_tools.py b/src/code4me2_agent/mcp_tools.py new file mode 100644 index 0000000..d86ff10 --- /dev/null +++ b/src/code4me2_agent/mcp_tools.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import re +from contextlib import AsyncExitStack +from dataclasses import dataclass +from pathlib import Path +from threading import Event, Thread +from typing import Any, Callable + +from mcp import Client, StdioServerParameters + +from code4me2_agent.acp_utils import capability_value + +logger = logging.getLogger(__name__) + +_MCP_STARTUP_TIMEOUT_SECONDS = 30.0 +_MCP_CALL_TIMEOUT_SECONDS = 120.0 +_OPENAI_TOOL_NAME_LIMIT = 64 +_UNSAFE_TOOL_NAME_CHARACTER_RE = re.compile(r"[^A-Za-z0-9_-]+") + + +@dataclass(frozen=True) +class _McpTool: + public_name: str + server_name: str + remote_name: str + definition: dict[str, Any] + + +class StdioMcpToolBroker: + """Own stdio MCP clients and expose their tools to the synchronous agent loop. + + MCP clients use AnyIO task groups whose context must be exited by the task that + entered it. A dedicated owner thread keeps that lifecycle intact while the + model/tool loop, which already runs in a worker thread, submits tool calls to + the owner's event loop. + """ + + def __init__( + self, + servers: list[object], + *, + cwd: Path, + client_factory: Callable[[StdioServerParameters], Client] = Client, + ) -> None: + self._servers = list(servers) + self._cwd = cwd + self._client_factory = client_factory + self._ready = Event() + self._closed = Event() + self._loop: asyncio.AbstractEventLoop | None = None + self._close_requested: asyncio.Event | None = None + self._startup_error: BaseException | None = None + self._tools: dict[str, _McpTool] = {} + self._clients: dict[str, Client] = {} + self._thread = Thread( + target=self._thread_main, + name="code4me2-mcp", + daemon=True, + ) + + @classmethod + def open( + cls, + servers: list[object], + *, + cwd: Path, + client_factory: Callable[[StdioServerParameters], Client] = Client, + startup_timeout_seconds: float = _MCP_STARTUP_TIMEOUT_SECONDS, + ) -> StdioMcpToolBroker | None: + if not servers: + return None + broker = cls(servers, cwd=cwd, client_factory=client_factory) + broker._thread.start() + if not broker._ready.wait(timeout=startup_timeout_seconds): + broker.close() + raise TimeoutError("Timed out while starting an ACP-provided MCP server.") + if broker._startup_error is not None: + broker.close() + raise RuntimeError( + f"Could not start an ACP-provided MCP server: {broker._startup_error}" + ) from broker._startup_error + return broker + + def definitions(self) -> list[dict[str, Any]]: + return [tool.definition for tool in self._tools.values()] + + def has_tool(self, name: str) -> bool: + return name in self._tools + + def execute(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + tool = self._tools.get(name) + if tool is None: + raise KeyError(f"Unknown MCP tool: {name}") + loop = self._loop + if loop is None or loop.is_closed() or self._closed.is_set(): + raise RuntimeError(f"MCP server {tool.server_name!r} is no longer available.") + future = asyncio.run_coroutine_threadsafe( + self._call_tool(tool, arguments), + loop, + ) + try: + return future.result(timeout=_MCP_CALL_TIMEOUT_SECONDS) + except TimeoutError: + future.cancel() + raise TimeoutError( + f"MCP tool {tool.remote_name!r} on server {tool.server_name!r} timed out." + ) from None + + def close(self) -> None: + if not self._thread.is_alive(): + return + loop = self._loop + close_requested = self._close_requested + if loop is not None and close_requested is not None and not loop.is_closed(): + loop.call_soon_threadsafe(close_requested.set) + self._thread.join(timeout=5.0) + if self._thread.is_alive(): + logger.warning("An ACP-provided MCP server did not stop within five seconds.") + + def _thread_main(self) -> None: + try: + asyncio.run(self._run()) + except BaseException as exc: # noqa: BLE001 + self._startup_error = exc + self._ready.set() + finally: + self._closed.set() + + async def _run(self) -> None: + self._loop = asyncio.get_running_loop() + self._close_requested = asyncio.Event() + async with AsyncExitStack() as stack: + await self._connect_servers(stack) + self._ready.set() + await self._close_requested.wait() + + async def _connect_servers(self, stack: AsyncExitStack) -> None: + for server in self._servers: + server_name, parameters = _stdio_server_parameters(server, cwd=self._cwd) + if server_name in self._clients: + raise ValueError(f"Duplicate ACP MCP server name: {server_name!r}") + client = await stack.enter_async_context(self._client_factory(parameters)) + self._clients[server_name] = client + await self._discover_tools(server_name, client) + + async def _discover_tools(self, server_name: str, client: Client) -> None: + cursor: str | None = None + while True: + result = await client.list_tools(cursor=cursor) + for tool in result.tools: + remote_name = str(tool.name) + public_name = _public_tool_name(server_name, remote_name) + if public_name in self._tools: + raise ValueError( + f"ACP MCP tools map to the same public name: {public_name!r}" + ) + parameters = dict(tool.input_schema or {"type": "object"}) + description = str(tool.description or tool.title or remote_name) + self._tools[public_name] = _McpTool( + public_name=public_name, + server_name=server_name, + remote_name=remote_name, + definition={ + "type": "function", + "function": { + "name": public_name, + "description": f"MCP server {server_name}: {description}", + "parameters": parameters, + }, + }, + ) + cursor = result.next_cursor + if cursor is None: + break + + async def _call_tool( + self, + tool: _McpTool, + arguments: dict[str, Any], + ) -> dict[str, Any]: + client = self._clients[tool.server_name] + result = await client.call_tool(tool.remote_name, arguments) + payload = result.model_dump(mode="json", by_alias=True, exclude_none=True) + return { + "status": "failed" if result.is_error else "completed", + "server_name": tool.server_name, + "tool_name": tool.remote_name, + "content": payload.get("content", []), + "structured_content": payload.get("structuredContent"), + } + + +def _stdio_server_parameters( + server: object, + *, + cwd: Path, +) -> tuple[str, StdioServerParameters]: + name = str(capability_value(server, "name") or "").strip() + command = str(capability_value(server, "command") or "").strip() + if not name or not command: + raise ValueError( + "Only named stdio MCP servers are supported; HTTP, SSE, and ACP transports " + "must not be sent when their capabilities are not advertised." + ) + args_value = capability_value(server, "args") or [] + env_value = capability_value(server, "env") or [] + if not isinstance(args_value, (list, tuple)) or not isinstance( + env_value, (list, tuple, dict) + ): + raise ValueError(f"Invalid stdio MCP configuration for server {name!r}.") + if isinstance(env_value, dict): + environment = {str(key): str(value) for key, value in env_value.items()} + else: + environment = {} + for item in env_value: + key = str(capability_value(item, "name") or "").strip() + value = capability_value(item, "value") + if not key or value is None: + raise ValueError(f"Invalid environment entry for MCP server {name!r}.") + environment[key] = str(value) + return name, StdioServerParameters( + command=command, + args=[str(value) for value in args_value], + env=environment, + cwd=cwd, + ) + + +def _public_tool_name(server_name: str, remote_name: str) -> str: + raw_name = f"mcp__{server_name}__{remote_name}" + normalized = _UNSAFE_TOOL_NAME_CHARACTER_RE.sub("_", raw_name).strip("_") + if not normalized: + normalized = "mcp_tool" + if len(normalized) <= _OPENAI_TOOL_NAME_LIMIT: + return normalized + digest = hashlib.sha256(raw_name.encode("utf-8")).hexdigest()[:10] + prefix_length = _OPENAI_TOOL_NAME_LIMIT - len(digest) - 2 + return f"{normalized[:prefix_length]}__{digest}" + + +def serialize_mcp_servers(servers: list[object] | None) -> str: + """Return a log-safe description without exposing environment values.""" + + summaries = [] + for server in servers or []: + summaries.append( + { + "name": capability_value(server, "name"), + "transport": ( + "stdio" if capability_value(server, "command") else "unsupported" + ), + } + ) + return json.dumps(summaries, sort_keys=True) diff --git a/tests/code4me2_agent/__init__.py b/tests/code4me2_agent/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/code4me2_agent/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/code4me2_agent/mcp_toy_server.py b/tests/code4me2_agent/mcp_toy_server.py new file mode 100644 index 0000000..a8f28ec --- /dev/null +++ b/tests/code4me2_agent/mcp_toy_server.py @@ -0,0 +1,23 @@ +"""Minimal stdio MCP server used to prove the broker attach path. + +Exposes a single ``toy_add`` tool. Run as a subprocess; speaks MCP over +stdio, so nothing is printed here. +""" + +from __future__ import annotations + +import asyncio + +from mcp.server.mcpserver import MCPServer + +server = MCPServer(name="toy") + + +@server.tool() +def toy_add(a: int, b: int) -> int: + """Add two integers.""" + return a + b + + +if __name__ == "__main__": + asyncio.run(server.run_stdio_async()) diff --git a/tests/code4me2_agent/test_acp_runtime.py b/tests/code4me2_agent/test_acp_runtime.py new file mode 100644 index 0000000..2f8704d --- /dev/null +++ b/tests/code4me2_agent/test_acp_runtime.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import asyncio +import sys +from importlib.metadata import version +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import Event +from typing import Any +from unittest import TestCase + +from acp.agent.router import build_agent_router +from acp.exceptions import RequestError +from acp.schema import TextContentBlock + +from code4me2_agent.acp_runtime import create_acp_agent +from code4me2_agent.acp_updates import AcpUpdateBuilder +from code4me2_agent.config import AgentConfig +from code4me2_agent.echo import EchoPromptResult +from code4me2_agent.runtime_auth import AcpRuntimeScope + + +class _FakeAuthorization: + def __init__(self, workspace: Path) -> None: + self.workspace = workspace.resolve() + self.is_authenticated = True + self.server_agent_config = None + self.backend_url = None + + def authenticate(self) -> AcpRuntimeScope: + return AcpRuntimeScope(project_id="project-1", workspace=str(self.workspace)) + + def validate(self) -> AcpRuntimeScope: + return self.authenticate() + + def telemetry_headers(self) -> dict[str, str]: + return {} + + def authorized_json_request( + self, + method: str, + path: str, + payload: object, + ) -> dict[str, object]: + if method == "GET": + return {"messages": []} + return {} + + +class _FakeClient: + def __init__(self) -> None: + self.updates: list[dict[str, Any]] = [] + + async def session_update( + self, + *, + session_id: str, + update: object, + **kwargs: Any, + ) -> None: + self.updates.append( + {"session_id": session_id, "update": update, "metadata": kwargs} + ) + + +class AcpRuntimeCompatibilityTest(TestCase): + def setUp(self) -> None: + self.temp_dir = TemporaryDirectory() + self.workspace = Path(self.temp_dir.name).resolve() + self.client = _FakeClient() + self.authorization = _FakeAuthorization(self.workspace) + self.agent = create_acp_agent( + AgentConfig( + workspace_root=self.workspace, + trace_path=self.workspace / "agent-events.jsonl", + session_id="bootstrap", + ), + authorization=self.authorization, + ) + self.agent.on_connect(self.client) + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def test_latest_sdk_models_and_stable_capabilities(self) -> None: + async def scenario() -> None: + response = await self.agent.initialize( + protocol_version=99, + client_capabilities=None, + ) + + self.assertEqual("0.12.1", version("agent-client-protocol")) + self.assertEqual(1, response.protocol_version) + self.assertFalse(response.agent_capabilities.load_session) + self.assertIsNotNone(response.agent_capabilities.session_capabilities.resume) + self.assertIsNotNone(response.agent_capabilities.session_capabilities.close) + self.assertEqual("AuthMethodAgent", type(response.auth_methods[0]).__name__) + self.assertEqual((), AcpUpdateBuilder().helper_gaps) + + asyncio.run(scenario()) + + def test_unsupported_routes_are_method_not_found_even_with_sdk_workaround(self) -> None: + async def scenario() -> None: + router = build_agent_router(self.agent, use_unstable_protocol=True) + + for method, params in ( + ("session/list", {}), + ("session/load", {"cwd": str(self.workspace), "sessionId": "missing"}), + ("session/set_model", {"sessionId": "missing", "modelId": "other"}), + ("session/mode", {"sessionId": "missing", "modeId": "other"}), + ( + "session/fork", + { + "cwd": str(self.workspace), + "sessionId": "missing", + "mcpServers": [], + }, + ), + ): + with self.subTest(method=method): + with self.assertRaises(RequestError) as raised: + await router(method, params, False) + self.assertEqual(-32601, raised.exception.code) + + asyncio.run(scenario()) + + def test_nonempty_unsupported_session_inputs_are_rejected(self) -> None: + async def scenario() -> None: + await self.agent.initialize(protocol_version=1, client_capabilities=None) + + with self.assertRaises(RequestError) as mcp_error: + await self.agent.new_session( + cwd=str(self.workspace), + mcp_servers=[{"name": "tools", "command": "tools-server"}], + ) + self.assertEqual("mcp_servers_not_supported", mcp_error.exception.data["reason"]) + + with self.assertRaises(RequestError) as roots_error: + await self.agent.new_session( + cwd=str(self.workspace), + additional_directories=[str(self.workspace / "other")], + ) + self.assertEqual( + "additional_directories_not_supported", + roots_error.exception.data["reason"], + ) + + asyncio.run(scenario()) + + def test_wellformed_but_unstartable_mcp_server_degrades_gracefully(self) -> None: + async def scenario() -> None: + await self.agent.initialize(protocol_version=1, client_capabilities=None) + new_session = await self.agent.new_session( + cwd=str(self.workspace), + mcp_servers=[ + { + "name": "missing", + "command": "/nonexistent-mcp-binary-xyz", + "args": [], + "env": [], + } + ], + ) + session = self.agent._sessions[new_session.session_id] + self.assertIsNone(session.mcp_tools) + + asyncio.run(scenario()) + + def test_working_stdio_mcp_server_attaches_tools(self) -> None: + async def scenario() -> None: + await self.agent.initialize(protocol_version=1, client_capabilities=None) + toy_path = str(Path(__file__).with_name("mcp_toy_server.py")) + new_session = await self.agent.new_session( + cwd=str(self.workspace), + mcp_servers=[ + { + "name": "toy", + "command": sys.executable, + "args": [toy_path], + "env": [], + } + ], + ) + session = self.agent._sessions[new_session.session_id] + try: + self.assertIsNotNone(session.mcp_tools) + names = [d["function"]["name"] for d in session.mcp_tools.definitions()] + self.assertIn("mcp__toy__toy_add", names) + result = session.mcp_tools.execute( + "mcp__toy__toy_add", {"a": 1, "b": 2} + ) + self.assertEqual("completed", result["status"]) + finally: + await self.agent.close_session(session_id=new_session.session_id) + + asyncio.run(scenario()) + + def test_active_prompt_cancellation_is_cooperative_and_returns_cancelled(self) -> None: + async def scenario() -> None: + await self.agent.initialize(protocol_version=1, client_capabilities=None) + new_session = await self.agent.new_session(cwd=str(self.workspace)) + session = self.agent._sessions[new_session.session_id] + prompt_started = Event() + + def blocking_prompt( + prompt: str, + request_id: str | None = None, + message_id: str | None = None, + run_id: str | None = None, + cancellation_event: Event | None = None, + ) -> EchoPromptResult: + prompt_started.set() + if cancellation_event is None or not cancellation_event.wait(timeout=2): + raise AssertionError("Cancellation was not propagated to the active turn") + return EchoPromptResult( + run_id=run_id or "cancelled-run", + final_response="", + stop_reason="cancelled", + run_status="cancelled", + ) + + session.core.handle_prompt = blocking_prompt + prompt_task = asyncio.create_task( + self.agent.prompt( + prompt=[TextContentBlock(type="text", text="wait")], + session_id=new_session.session_id, + ) + ) + started = await asyncio.to_thread(prompt_started.wait, 1) + self.assertTrue(started) + + await self.agent.cancel(session_id=new_session.session_id) + response = await asyncio.wait_for(prompt_task, timeout=2) + + self.assertEqual("cancelled", response.stop_reason) + self.assertEqual([], self.client.updates) + self.assertIsNone(session.active_prompt_task) + self.assertFalse(session.cancel_event.is_set()) + + asyncio.run(scenario()) + + def test_prompt_message_has_stable_message_id_and_close_releases_session(self) -> None: + async def scenario() -> None: + await self.agent.initialize(protocol_version=1, client_capabilities=None) + new_session = await self.agent.new_session(cwd=str(self.workspace)) + + response = await self.agent.prompt( + prompt=[TextContentBlock(type="text", text="hello")], + session_id=new_session.session_id, + ) + + self.assertEqual("end_turn", response.stop_reason) + self.assertEqual(1, len(self.client.updates)) + update = self.client.updates[0]["update"] + self.assertIsNotNone(update.message_id) + + await self.agent.close_session(session_id=new_session.session_id) + self.assertNotIn(new_session.session_id, self.agent._sessions) + + asyncio.run(scenario()) From 038dbfcf66f4f1b1738e504ef1c6c6cbb4eac929 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Thu, 10 Sep 2026 11:52:56 +0300 Subject: [PATCH 005/115] Fetch agent profile once per auth instead of per message --- src/code4me2_agent/runtime_auth.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/code4me2_agent/runtime_auth.py b/src/code4me2_agent/runtime_auth.py index f1a4824..f1060a3 100644 --- a/src/code4me2_agent/runtime_auth.py +++ b/src/code4me2_agent/runtime_auth.py @@ -97,7 +97,9 @@ def authenticate(self) -> AcpRuntimeScope: self._refresh_prepared_credentials() if self._backend_url and self._acp_token: try: - return self.validate() + scope = self.validate() + self.fetch_agent_config() + return scope except AcpAuthorizationFailure: logger.info( "Cached ACP runtime token was not valid; falling back to prepared grant exchange." @@ -249,13 +251,13 @@ def server_store_agent_content(self) -> bool: def fetch_agent_config(self) -> None: """Fetch the assigned agent profile's runtime configuration. - Called right after a successful grant exchange. A failure here is - logged and tolerated: the agent then runs on whatever the local config - file provides, which is a degraded but working state rather than a dead - one. Note that with the provider defaults removed, a local config that - specifies no model will fail loudly at first use instead — which is the - intended outcome, since silently substituting a different model would - corrupt the experiment. + Called once after a successful grant exchange or cached-token + validation. A failure here is logged and tolerated: the agent then runs + on whatever the local config file provides, which is a degraded but + working state rather than a dead one. Note that with the provider + defaults removed, a local config that specifies no model will fail + loudly at first use instead — which is the intended outcome, since + silently substituting a different model would corrupt the experiment. """ if not self._backend_url or not self._acp_token: logger.info("Cannot fetch agent config: not authenticated.") From 52c93cf9ff58041a9f8e8779cf901ebf13ed34d4 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Thu, 10 Sep 2026 20:18:46 +0200 Subject: [PATCH 006/115] feat(acp): route authenticated agents through backend relay Fetch agent configuration after authorization and keep provider credentials on the backend. --- docker-compose.dev-arm.yaml | 6 ++-- docker-compose.yml | 3 +- src/backend/routers/acp/__init__.py | 47 ++++++++++++++++++++++++++++- src/code4me2_agent/adapters.py | 2 +- src/code4me2_agent/config.py | 17 +++-------- src/code4me2_agent/runtime_auth.py | 1 + 6 files changed, 58 insertions(+), 18 deletions(-) diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml index 81eee29..0d7ce3a 100644 --- a/docker-compose.dev-arm.yaml +++ b/docker-compose.dev-arm.yaml @@ -115,9 +115,9 @@ services: "CMD", "bash", "-c", - "source activate myenv && celery -A celery_app.celery_app status", + "source activate myenv && celery -A celery_app.celery_app status --timeout=5", ] - start_period: 0s + start_period: 30s interval: 60s timeout: 10s retries: 3 @@ -211,7 +211,7 @@ services: website: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost"] + test: ["CMD", "curl", "-f", "http://localhost:8000"] start_period: 0s interval: 60s timeout: 5s diff --git a/docker-compose.yml b/docker-compose.yml index e996ced..b3bac15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: volumes: - .:/app - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql + - postgres_data:/var/lib/postgresql/data environment: - POSTGRES_PASSWORD=${DB_PASSWORD} - POSTGRES_USER=${DB_USER} @@ -265,7 +266,7 @@ services: website: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost"] + test: ["CMD", "curl", "-f", "http://localhost:8000"] start_period: 0s interval: 60s timeout: 5s diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py index 95bb8f2..a15741f 100644 --- a/src/backend/routers/acp/__init__.py +++ b/src/backend/routers/acp/__init__.py @@ -27,10 +27,12 @@ import uuid from typing import Optional -from fastapi import APIRouter, Cookie, Depends, Header +import httpx +from fastapi import APIRouter, Body, Cookie, Depends, Header, HTTPException, Response import Queries from App import App +from agents import provider as provider_module from backend.acp_authorization import ( AcpAuthorizationDenied, AcpAuthorizationService, @@ -45,6 +47,8 @@ ValidateAcpSessionPostResponse, ) from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from backend.routers.agent.acp_auth import require_acp_scope +from agents import registry from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin from database import crud from utils import create_uuid @@ -64,6 +68,47 @@ _BEARER_PREFIX = "Bearer " +@router.post("/chat/completions") +async def acp_chat_completions( + body: dict = Body(...), + scope=Depends(require_acp_scope), + app: App = Depends(App.get_instance), +) -> Response: + """Proxy a built-in agent turn without exposing provider credentials locally.""" + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for ACP inference.") + + db = app.get_db_session() + try: + profile = registry.resolve_assignment(db, uuid.UUID(scope.user_id)) + finally: + db.close() + if profile is None: + raise HTTPException(status_code=503, detail="No active agent profile is configured.") + + payload = dict(body) + payload["model"] = profile.model + if profile.temperature is not None: + payload["temperature"] = profile.temperature + upstream = provider_module.resolve_upstream( + model=profile.model, + base_url=profile.base_url, + api_key_ref=profile.api_key_ref, + framework_version=profile.framework_version, + ) + async with httpx.AsyncClient(timeout=120) as client: + upstream_response = await client.post( + upstream.endpoint(responses_api=False), + json=payload, + headers={"Authorization": f"Bearer {upstream.api_key}"}, + ) + return Response( + content=upstream_response.content, + status_code=upstream_response.status_code, + media_type=upstream_response.headers.get("content-type", "application/json"), + ) + + def _bearer_token(authorization: str) -> Optional[str]: if not authorization.startswith(_BEARER_PREFIX): return None diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py index 1ee17a3..39eb571 100644 --- a/src/code4me2_agent/adapters.py +++ b/src/code4me2_agent/adapters.py @@ -540,7 +540,7 @@ def _headers(self) -> dict[str, str]: def _openai_base_url(self) -> str: if self._kind == "code4me_backend": - return f"{self._base_url}/api" + return f"{self._base_url}/api/acp" base_url = self._base_url if not base_url.endswith("/v1"): base_url = f"{base_url}/v1" diff --git a/src/code4me2_agent/config.py b/src/code4me2_agent/config.py index 83b42f2..f381f79 100644 --- a/src/code4me2_agent/config.py +++ b/src/code4me2_agent/config.py @@ -197,22 +197,15 @@ def with_server_overrides( runtime. Precedence is deliberate: **server wins**, and the local config file is only a fallback for anything the server didn't specify. - Routing follows from whether the profile named a ``base_url``: - - * it did → talk to that endpoint directly (``kind="openai"``), which is - what makes "any OpenAI-compatible provider" work per merge decision 6; - * it didn't → relay through the backend (``kind="code4me_backend"``), so - the backend chooses the upstream and also observes the call. + An authenticated runtime always relays through the backend. Provider + credentials stay in the backend environment; the local agent holds + only its short-lived ACP bearer token. """ from dataclasses import replace as _replace provider = self.adapter.provider - if server.base_url: - provider = _replace( - provider, kind="openai", base_url=server.base_url - ) - elif backend_url: + if backend_url: provider = _replace( provider, kind="code4me_backend", @@ -221,7 +214,7 @@ def with_server_overrides( if server.model: provider = _replace(provider, model=server.model) - if server.api_key_ref: + if server.api_key_ref and provider.kind != "code4me_backend": provider = _replace(provider, api_key_env=server.api_key_ref) memory_window = self.adapter.memory_window diff --git a/src/code4me2_agent/runtime_auth.py b/src/code4me2_agent/runtime_auth.py index f1060a3..76b030e 100644 --- a/src/code4me2_agent/runtime_auth.py +++ b/src/code4me2_agent/runtime_auth.py @@ -186,6 +186,7 @@ def validate(self) -> AcpRuntimeScope: project_id, workspace, ) + self.fetch_agent_config() return self._scope # The assigned agent profile, as handed down by the backend after auth. From 34a4a2cc3d843ba56aeb15780089bea5a36bc1f2 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Thu, 10 Sep 2026 22:27:09 +0200 Subject: [PATCH 007/115] fix: use the configured allowlisted tools in code4me-agent and return tool failure back to agent --- src/code4me2_agent/acp_runtime.py | 8 +++ src/code4me2_agent/adapters.py | 26 ++++++- src/code4me2_agent/config.py | 7 +- tests/code4me2_agent/test_acp_runtime.py | 90 +++++++++++++++++++++++- 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py index 55571c1..4ed3017 100644 --- a/src/code4me2_agent/acp_runtime.py +++ b/src/code4me2_agent/acp_runtime.py @@ -785,6 +785,11 @@ def _apply_config_to_all_cores(self, new_config: Any) -> None: new_config.commands.allowlisted_commands ) self._bootstrap_core.file_tools._config = new_config + bootstrap_registry = getattr( + self._bootstrap_core._adapter, "_tool_registry", None + ) + if bootstrap_registry is not None: + bootstrap_registry.set_allowed_tools(new_config.tools) for session_id, session in self._sessions.items(): session.core._config = new_config session.core._adapter._config = new_config @@ -793,6 +798,9 @@ def _apply_config_to_all_cores(self, new_config: Any) -> None: new_config.commands.allowlisted_commands ) session.core.file_tools._config = new_config + tool_registry = getattr(session.core._adapter, "_tool_registry", None) + if tool_registry is not None: + tool_registry.set_allowed_tools(new_config.tools) logger.info( "Updated %d active session(s) with server agent config.", len(self._sessions), diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py index 39eb571..34b4c5d 100644 --- a/src/code4me2_agent/adapters.py +++ b/src/code4me2_agent/adapters.py @@ -230,6 +230,7 @@ def __init__( file_tools: WorkspaceFileTools, command_tools: WorkspaceCommandTools, *, + allowed_tools: list[str] | None = None, event_sink: AgentEventSink | None = None, mcp_tools: StdioMcpToolBroker | None = None, ) -> None: @@ -242,6 +243,7 @@ def __init__( self._file_tools = file_tools self._command_tools = command_tools self._mcp_tools = mcp_tools + self._allowed_tools = set(allowed_tools) if allowed_tools is not None else None self._event_sink = event_sink or NoopAgentEventSink() self._event_type = ToolCallEvent self._thought_event_type = ThoughtEvent @@ -250,6 +252,11 @@ def execute( self, tool_call: ToolCall, *, run_id: str, request_id: str ) -> dict[str, Any]: name = tool_call.name + if self._allowed_tools is not None and name not in self._allowed_tools: + raise ToolRegistryError( + f"Tool is not allowed by the assigned profile: {name}", + failure_reason="tool_not_allowed", + ) arguments = dict(tool_call.arguments) self._emit_tool_start( tool_call, arguments, run_id=run_id, request_id=request_id @@ -334,8 +341,17 @@ def definitions(self) -> list[dict[str, Any]]: definitions = _tool_definitions() if self._mcp_tools is not None: definitions.extend(self._mcp_tools.definitions()) + if self._allowed_tools is not None: + definitions = [ + tool + for tool in definitions + if tool.get("function", {}).get("name") in self._allowed_tools + ] return definitions + def set_allowed_tools(self, allowed_tools: list[str] | None) -> None: + self._allowed_tools = set(allowed_tools) if allowed_tools is not None else None + def known_tool_names(self) -> set[str]: names: set[str] = set() for tool in self.definitions(): @@ -791,14 +807,19 @@ def handle_prompt( request_id=request_id, ) except ToolRegistryError as exc: - return self._tool_failure_result( + self._record_tool_failure( run_id=run_id, request_id=request_id, tool_call=tool_call, failure_reason=exc.failure_reason, error_message=str(exc), - thoughts=thoughts, ) + tool_output = { + "status": "denied", + "error": str(exc), + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } except PermissionError as exc: tool_output = { "status": "denied", @@ -1265,6 +1286,7 @@ def create_agent_adapter( tool_registry=ToolRegistry( file_tools, command_tools, + allowed_tools=config.tools, event_sink=event_sink, mcp_tools=mcp_tools, ), diff --git a/src/code4me2_agent/config.py b/src/code4me2_agent/config.py index f381f79..01bf93a 100644 --- a/src/code4me2_agent/config.py +++ b/src/code4me2_agent/config.py @@ -183,6 +183,7 @@ class AgentConfig: workspace_root: Path trace_path: Path session_id: str + tools: list[str] | None = None raw_capture_enabled: bool = False upload: UploadConfig = field(default_factory=UploadConfig) commands: CommandConfig = field(default_factory=CommandConfig) @@ -236,7 +237,11 @@ def with_server_overrides( commands, allowlisted_commands=list(server.commands_allowlist) ) - return _replace(self, adapter=adapter, commands=commands) + tools = self.tools + if server.tools is not None: + tools = list(server.tools) + + return _replace(self, adapter=adapter, commands=commands, tools=tools) @classmethod def from_file(cls, config_path: str | Path) -> "AgentConfig": diff --git a/tests/code4me2_agent/test_acp_runtime.py b/tests/code4me2_agent/test_acp_runtime.py index 2f8704d..baac667 100644 --- a/tests/code4me2_agent/test_acp_runtime.py +++ b/tests/code4me2_agent/test_acp_runtime.py @@ -13,11 +13,23 @@ from acp.exceptions import RequestError from acp.schema import TextContentBlock +from code4me2_agent.adapters import ( + OpenAICompatibleReactAdapter, + ToolCall, + ToolRegistry, + ToolRegistryError, +) from code4me2_agent.acp_runtime import create_acp_agent from code4me2_agent.acp_updates import AcpUpdateBuilder -from code4me2_agent.config import AgentConfig +from code4me2_agent.config import ( + AdapterConfig, + AgentConfig, + FakeProviderConfig, + ServerAgentConfig, +) from code4me2_agent.echo import EchoPromptResult from code4me2_agent.runtime_auth import AcpRuntimeScope +from code4me2_agent.telemetry import AgentTelemetryRecorder class _FakeAuthorization: @@ -99,6 +111,82 @@ async def scenario() -> None: asyncio.run(scenario()) + def test_server_tool_allowlist_hides_and_rejects_write_tools(self) -> None: + config = AgentConfig( + workspace_root=self.workspace, + trace_path=self.workspace / "agent-events.jsonl", + session_id="bootstrap", + ).with_server_overrides(ServerAgentConfig(tools=["read_file"])) + registry = ToolRegistry( + file_tools=object(), + command_tools=object(), + allowed_tools=config.tools, + ) + + self.assertEqual( + ["read_file"], + [tool["function"]["name"] for tool in registry.definitions()], + ) + with self.assertRaises(ToolRegistryError) as raised: + registry.execute( + ToolCall( + tool_call_id="write-denied", + name="write_file", + arguments={"path": "deneme.txt", "content": "hey"}, + ), + run_id="run-1", + request_id="request-1", + ) + self.assertEqual("tool_not_allowed", raised.exception.failure_reason) + + def test_denied_tool_call_continues_with_a_tool_result(self) -> None: + config = AgentConfig( + workspace_root=self.workspace, + trace_path=self.workspace / "agent-events.jsonl", + session_id="bootstrap", + tools=["read_file"], + adapter=AdapterConfig( + name="openai_compatible_react", + fake_provider=FakeProviderConfig( + enabled=True, + script=[ + { + "tool_calls": [ + { + "id": "write-denied", + "name": "write_file", + "arguments": { + "path": "deneme.txt", + "content": "hi", + }, + } + ] + }, + {"final_answer": "I only have read access in this profile."}, + ], + ), + ), + ) + adapter = OpenAICompatibleReactAdapter( + config, + telemetry=AgentTelemetryRecorder(config), + tool_registry=ToolRegistry( + file_tools=object(), + command_tools=object(), + allowed_tools=config.tools, + ), + ) + + result = adapter.handle_prompt( + prompt="Write hi in deneme.txt", + run_id="run-1", + request_id="request-1", + message_id=None, + ) + + self.assertEqual("completed", result.run_status) + self.assertEqual("I only have read access in this profile.", result.final_response) + def test_unsupported_routes_are_method_not_found_even_with_sdk_workaround(self) -> None: async def scenario() -> None: router = build_agent_router(self.agent, use_unstable_protocol=True) From 82013f6e5d1c18f2eaae73555e188347b3b05d64 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Fri, 11 Sep 2026 12:36:13 +0300 Subject: [PATCH 008/115] feat: managed agent policy backend and schema - Add f3c4d5e6a7b9 policy-snapshot migration, crud/schemas/manager support - Add ACP managed endpoints (capabilities/readiness/runs/inference) alongside chat/completions relay - Enforce policy fail-closed, provider creds stay server-side - Add managed backend tests --- src/Queries.py | 20 +- src/agents/inference.py | 47 +- src/agents/tools.py | 3 + src/backend/Responses.py | 11 + src/backend/acp_authorization.py | 98 +++- src/backend/completion/.env | 6 +- src/backend/routers/acp/__init__.py | 490 +++++++++++++++++- src/backend/routers/agent/ingest.py | 65 ++- src/backend/routers/agent/profiles.py | 61 ++- src/database/crud.py | 2 + src/database/db_schemas.py | 4 +- src/database/migration/migration_manager.py | 100 +++- ...6a7b9_add_managed_agent_policy_snapshot.py | 27 + tests/backend_tests/test_managed_acp.py | 486 +++++++++++++++++ .../database_tests/test_migration_cli_exit.py | 29 ++ 15 files changed, 1379 insertions(+), 70 deletions(-) create mode 100644 src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py create mode 100644 tests/backend_tests/test_managed_acp.py create mode 100644 tests/database_tests/test_migration_cli_exit.py diff --git a/src/Queries.py b/src/Queries.py index 28acf7b..5bd63be 100644 --- a/src/Queries.py +++ b/src/Queries.py @@ -394,7 +394,25 @@ class PrepareAcpGrant(QueryBase): project_id: UUID = Field( ..., description="Activated project id for the ACP process" ) - workspace: str = Field(..., description="Canonical project workspace", min_length=1) + workspace: str = Field( + ..., description="Canonical project workspace", min_length=1, max_length=4096 + ) + launch_id: Optional[str] = Field( + default=None, + description="Stable identity for one managed runtime launch.", + min_length=1, + max_length=200, + ) + path_format: Optional[str] = Field( + default=None, + description="Client workspace syntax: posix or windows.", + pattern="^(posix|windows)$", + ) + managed_protocol_version: Optional[str] = Field( + default=None, + description="Managed-agent protocol requested by the plugin.", + pattern="^1$", + ) class ExchangeAcpGrant(QueryBase): diff --git a/src/agents/inference.py b/src/agents/inference.py index 5654848..ec4c18f 100644 --- a/src/agents/inference.py +++ b/src/agents/inference.py @@ -27,7 +27,6 @@ import time import uuid from collections import Counter -from datetime import datetime from typing import TYPE_CHECKING, Optional import httpx @@ -55,6 +54,8 @@ from database import crud if TYPE_CHECKING: + from datetime import datetime + from App import App # Upstream request timeout. Agent turns with large contexts are slow, and a @@ -76,6 +77,7 @@ async def run_inference( framework_version: Optional[str] = None, content_included: bool = False, profile_tools_json: Optional[str] = None, + record_observation_events: bool = True, app: App, ) -> Response: """Forward one agent inference call upstream and record it as an agent_event. @@ -254,15 +256,16 @@ async def run_inference( tool_executions = extract_tool_executions( new_messages, is_responses_api=is_responses_api ) - write_tool_call_events( - app, - task_uuid, - chat_session_index, - tool_executions, - prev_call_at, - content_included, - parent_span_id=prev_model_call_span_id, - ) + if record_observation_events: + write_tool_call_events( + app, + task_uuid, + chat_session_index, + tool_executions, + prev_call_at, + content_included, + parent_span_id=prev_model_call_span_id, + ) system_content = next( ( @@ -305,11 +308,15 @@ async def run_inference( effective_allowlist = ( profile_tools if profile_tools is not None else KNOWN_AGENT_TOOLS ) - kept = [ - t - for t in original_tools - if t.get("function", {}).get("name") in effective_allowlist - ] + kept = [] + for tool in original_tools: + tool_name = tool.get("function", {}).get("name") + if tool_name in effective_allowlist or ( + "mcp__*" in effective_allowlist + and isinstance(tool_name, str) + and tool_name.startswith("mcp__") + ): + kept.append(tool) tools_stripped = len(original_tools) - len(kept) tools_kept = len(kept) for tool in kept: @@ -491,9 +498,10 @@ async def _stream(): captured_status[0] if captured_status else 0, ) _log_record(record) - write_model_call_event( - app, task_uuid, record, latency_ms, span, extra - ) + if record_observation_events: + write_model_call_event( + app, task_uuid, record, latency_ms, span, extra + ) return StreamingResponse(_stream(), media_type="text/event-stream") @@ -533,7 +541,8 @@ async def _stream(): upstream_resp.status_code, ) _log_record(record) - write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + if record_observation_events: + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) return Response( content=upstream_resp.content, diff --git a/src/agents/tools.py b/src/agents/tools.py index d1ba1b1..4e7506f 100644 --- a/src/agents/tools.py +++ b/src/agents/tools.py @@ -36,6 +36,9 @@ "list_files", "search_files", "run_command", + # Explicit opt-in for any tool exposed by client-supplied ACP MCP servers. + # Concrete names remain filtered and approval-gated by the managed runtime. + "mcp__*", } ) diff --git a/src/backend/Responses.py b/src/backend/Responses.py index 1999cda..fa1d493 100644 --- a/src/backend/Responses.py +++ b/src/backend/Responses.py @@ -243,6 +243,11 @@ class PrepareAcpGrantPostResponse(BaseResponse): grant: str = Field(..., description="One-time ACP launch grant.") workspace: str = Field(..., description="Authorized canonical workspace.") expires_in_seconds: int = Field(default=300, description="Grant lifetime.") + launch_id: Optional[str] = Field(default=None, description="Managed launch identity.") + path_format: Optional[str] = Field(default=None, description="Client path syntax.") + managed_protocol_version: Optional[str] = Field( + default=None, description="Negotiated managed-agent protocol." + ) class ExchangeAcpGrantPostResponse(BaseResponse): @@ -317,6 +322,12 @@ class AcpAgentConfigGetResponse(BaseResponse): "sends, so the agent may use it to avoid transmitting content " "needlessly but cannot use it to enable capture.", ) + transport: Optional[str] = Field( + default=None, description="Managed inference transport selected by the server." + ) + managed_protocol_version: Optional[str] = Field( + default=None, description="Negotiated managed-agent protocol." + ) class PersistentAuthTokenPostResponse(BaseResponse): diff --git a/src/backend/acp_authorization.py b/src/backend/acp_authorization.py index 4c5bc19..07d195d 100644 --- a/src/backend/acp_authorization.py +++ b/src/backend/acp_authorization.py @@ -26,7 +26,7 @@ import secrets from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Any, Callable, Optional, Protocol, Union @@ -66,6 +66,9 @@ class AcpAuthorizationDenied(Exception): class PreparedAcpGrant: grant: str workspace: str + launch_id: Optional[str] = None + path_format: Optional[str] = None + managed_protocol_version: Optional[str] = None expires_in_seconds: int = 300 @@ -135,6 +138,9 @@ def prepare_grant( auth_token: str, project_id: str, workspace: str, + launch_id: Optional[str] = None, + path_format: Optional[str] = None, + managed_protocol_version: Optional[str] = None, ) -> PreparedAcpGrant: """Mint a one-time launch grant for an authenticated plugin. @@ -142,23 +148,68 @@ def prepare_grant( so re-preparing (which the plugin does on every startup and login) leaves exactly one usable grant rather than accumulating valid tokens. """ + managed_values = (launch_id, path_format, managed_protocol_version) + if any(value is not None for value in managed_values) and not all( + value is not None for value in managed_values + ): + raise AcpAuthorizationDenied( + "Managed launch identity, path format, and protocol are required together." + ) + if managed_protocol_version is not None and managed_protocol_version != "1": + raise AcpAuthorizationDenied("Unsupported managed protocol version.") + canonical_workspace = ( + self._validate_client_workspace(workspace, path_format) + if path_format is not None + else self._canonical_workspace(workspace) + ) state = self._active_plugin_scope( auth_token=auth_token, project_id=project_id, - workspace=workspace, + workspace=canonical_workspace, + already_canonical=True, ) - grant = self._token_factory() - pending = self._tokens.get("acp_pending_grant", state["parent_session_token"]) + if launch_id is not None: + state["launch_id"] = launch_id + if path_format is not None: + state["path_format"] = path_format + if managed_protocol_version is not None: + state["managed_protocol_version"] = managed_protocol_version + pending_key = self._pending_grant_key(state, launch_id) + pending = self._tokens.get("acp_pending_grant", pending_key) if pending is not None and pending.get("grant"): - self._tokens.discard("acp_grant", pending["grant"]) + pending_grant = pending["grant"] + pending_state = self._tokens.get("acp_grant", pending_grant) + if launch_id is not None and pending_state == { + **state, + "pending_grant_key": pending_key, + }: + # A bridge request can be retried after its HTTP response is + # lost. Return the still-live grant for this exact launch so + # two concurrent responses cannot invalidate each other. + return PreparedAcpGrant( + grant=pending_grant, + workspace=state["workspace"], + launch_id=launch_id, + path_format=path_format, + managed_protocol_version=managed_protocol_version, + ) + self._tokens.discard("acp_grant", pending_grant) + grant = self._token_factory() + state["pending_grant_key"] = pending_key self._tokens.set("acp_grant", grant, state, force_reset_exp=True) self._tokens.set( "acp_pending_grant", - state["parent_session_token"], + pending_key, {"grant": grant}, force_reset_exp=True, ) - return PreparedAcpGrant(grant=grant, workspace=state["workspace"]) + return PreparedAcpGrant( + grant=grant, + workspace=state["workspace"], + launch_id=launch_id, + path_format=path_format, + managed_protocol_version=managed_protocol_version, + ) def exchange_grant(self, grant: str) -> AcpSessionAuthorization: """Trade a launch grant for an agent session token. @@ -178,6 +229,9 @@ def exchange_grant(self, grant: str) -> AcpSessionAuthorization: raise AcpAuthorizationDenied( "ACP grant is invalid or has already been used." ) + pending_key = consumed_state.get("pending_grant_key") + if pending_key: + self._tokens.discard("acp_pending_grant", pending_key) acp_token = self._token_factory() self._tokens.set("acp_session", acp_token, consumed_state, force_reset_exp=True) return AcpSessionAuthorization( @@ -237,6 +291,7 @@ def _active_plugin_scope( auth_token: str, project_id: str, workspace: str, + already_canonical: bool = False, ) -> dict[str, str]: auth_info = self._tokens.get("auth_token", auth_token) if auth_info is None or not auth_info.get("user_id"): @@ -245,12 +300,17 @@ def _active_plugin_scope( user_info = self._tokens.get("user_token", user_id) if user_info is None or not user_info.get("session_token"): raise AcpAuthorizationDenied("An active plugin session is required.") + # Managed grants arrive with a client-canonicalized workspace (validated + # against the participant OS without touching the server filesystem). + # Re-resolving that value with the server OS would reject Windows drive + # paths on a Linux backend, so already-canonical values are preserved. + canonical = workspace if already_canonical else self._canonical_workspace(workspace) return self._bound_scope( auth_token=auth_token, user_id=user_id, session_token=user_info["session_token"], project_id=project_id, - workspace=self._canonical_workspace(workspace), + workspace=canonical, ) def _require_active_scope(self, state: dict[str, str]) -> None: @@ -318,3 +378,25 @@ def _canonical_workspace(workspace: str) -> str: if not path.is_absolute(): raise AcpAuthorizationDenied("A canonical absolute workspace is required.") return path.resolve().as_posix() + + @staticmethod + def _validate_client_workspace(workspace: str, path_format: str) -> str: + """Validate an already-canonical client path without using the server OS.""" + if path_format not in {"windows", "posix"}: + raise AcpAuthorizationDenied("Unsupported client path format.") + path_cls = PureWindowsPath if path_format == "windows" else PurePosixPath + path = path_cls(workspace) + if not path.is_absolute() or ".." in path.parts: + raise AcpAuthorizationDenied("A canonical absolute workspace is required.") + return str(path) + + @staticmethod + def _pending_grant_key(state: dict[str, str], launch_id: Optional[str]) -> str: + # Legacy callers retain the old one-pending-grant-per-plugin-session + # behavior. Managed callers isolate retries and concurrent project launches. + if launch_id is None: + return state["parent_session_token"] + return ( + f"{state['parent_session_token']}:{state['project_id']}:" + f"{state['workspace']}:{launch_id}" + ) diff --git a/src/backend/completion/.env b/src/backend/completion/.env index fb62dbc..2bb153a 100644 --- a/src/backend/completion/.env +++ b/src/backend/completion/.env @@ -28,6 +28,10 @@ DB_PORT=2345 DB_USER=postgres DB_PASSWORD=postgres DB_NAME=code4meV2 +DB_POOL_SIZE=10 +DB_MAX_OVERFLOW=20 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE=1800 # ----------------------- # pgAdmin Configuration @@ -83,4 +87,4 @@ THREAD_POOL_MAX_WORKERS=2 # ----------------------- # Hugging Face Configuration HF_TOKEN=your_huggingface_token_here -# ----------------------- \ No newline at end of file +# ----------------------- diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py index a15741f..059a43a 100644 --- a/src/backend/routers/acp/__init__.py +++ b/src/backend/routers/acp/__init__.py @@ -24,18 +24,25 @@ import json import logging +import os import uuid +from datetime import datetime, timezone from typing import Optional import httpx -from fastapi import APIRouter, Body, Cookie, Depends, Header, HTTPException, Response +from fastapi import APIRouter, Body, Cookie, Depends, Header, HTTPException, Query, Response +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field +from sqlalchemy import text -import Queries +import Queries # noqa: TC001 - FastAPI evaluates route annotations at runtime +from agents.tools import CODE4ME2_AGENT_TOOLS from App import App from agents import provider as provider_module from backend.acp_authorization import ( AcpAuthorizationDenied, AcpAuthorizationService, + AcpServerAuthorization, ) from backend.Responses import ( AcpAgentConfigGetResponse, @@ -46,10 +53,14 @@ PrepareAcpGrantPostResponse, ValidateAcpSessionPostResponse, ) +from backend.routers.agent.acp_auth import require_acp_scope, require_acp_server_scope from backend.routers.agent.consent import resolve_store_agent_content_for_acp -from backend.routers.agent.acp_auth import require_acp_scope +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) from agents import registry -from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin from database import crud from utils import create_uuid @@ -64,6 +75,11 @@ FALLBACK_TOOLS = ["read_file", "list_files", "search_files"] FALLBACK_MODEL = "qwen2.5-coder:7b" FALLBACK_MAX_ITERATIONS = 6 +FALLBACK_MAX_CONTEXT_TOKENS = 16_000 +MANAGED_PROTOCOL_VERSION = "1" +MANAGED_RUNTIME = "code4me2-agent" +EXPECTED_SCHEMA_REVISION = "f3c4d5e6a7b9" +SUPPORTED_APPROVAL_POLICIES = frozenset({"auto", "per_step", "suggestion_only"}) _BEARER_PREFIX = "Bearer " @@ -109,6 +125,15 @@ async def acp_chat_completions( ) +class ManagedRunRequest(BaseModel): + run_id: str = Field(min_length=1, max_length=200) + session_id: str = Field(min_length=1, max_length=500) + + +class ManagedInferenceRequest(ManagedRunRequest): + request: dict + + def _bearer_token(authorization: str) -> Optional[str]: if not authorization.startswith(_BEARER_PREFIX): return None @@ -116,6 +141,47 @@ def _bearer_token(authorization: str) -> Optional[str]: return token or None +def _schema_revision(app: App) -> Optional[str]: + """Return the applied Alembic revision without relying on ORM metadata.""" + db = app.get_db_session() + try: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + except Exception as error: + logging.warning("[ACP/capabilities] schema revision unavailable: %s", error) + return None + finally: + db.close() + + +@router.get("/capabilities") +def get_acp_capabilities(app: App = Depends(App.get_instance)) -> dict: + """Advertise the managed-agent contract and whether its schema is deployed. + + Deployment readiness is gated on the expected Alembic revision so a + readiness probe can fail before participant traffic is enabled: when the + schema is missing or behind, this returns HTTP 503 instead of a 200 with + ``schema_ready=False``. + """ + revision = _schema_revision(app) + ready = revision == EXPECTED_SCHEMA_REVISION + payload = { + "managed_protocol_versions": [MANAGED_PROTOCOL_VERSION], + "supported_runtimes": [MANAGED_RUNTIME], + "schema_ready": ready, + "schema_revision": revision, + "expected_schema_revision": EXPECTED_SCHEMA_REVISION, + } + if not ready: + raise HTTPException( + status_code=503, + detail=( + f"Managed agent schema not ready: have {revision!r}, " + f"want {EXPECTED_SCHEMA_REVISION!r}" + ), + ) + return payload + + @router.post( "/persistent-auth-token", response_model=PersistentAuthTokenPostResponse, @@ -165,6 +231,9 @@ def prepare_acp_grant( auth_token=auth_token, project_id=str(request.project_id), workspace=request.workspace, + launch_id=request.launch_id, + path_format=request.path_format, + managed_protocol_version=request.managed_protocol_version, ) return JsonResponseWithStatus( status_code=201, @@ -172,6 +241,9 @@ def prepare_acp_grant( grant=prepared.grant, workspace=prepared.workspace, expires_in_seconds=prepared.expires_in_seconds, + launch_id=prepared.launch_id, + path_format=prepared.path_format, + managed_protocol_version=prepared.managed_protocol_version, ), ) except AcpAuthorizationDenied as error: @@ -252,6 +324,7 @@ def validate_or_refresh_acp_session( def get_acp_agent_config( app: App = Depends(App.get_instance), authorization: str = Header(default=""), + managed_protocol_version: Optional[str] = Query(default=None), ) -> JsonResponseWithStatus: """Return the runtime config for the calling agent process. @@ -282,8 +355,11 @@ def get_acp_agent_config( ) # Fallback defaults, overridden below where a profile/config supplies values. + # Managed callers never use these fallbacks (see managed branch below); + # they fail closed instead so a malformed study profile cannot start work. commands_allowlist = list(FALLBACK_COMMANDS_ALLOWLIST) tools = list(FALLBACK_TOOLS) + tools_parse_failed = False model = FALLBACK_MODEL max_iterations = FALLBACK_MAX_ITERATIONS agent_profile: Optional[str] = None @@ -325,7 +401,10 @@ def get_acp_agent_config( parsed_tools = json.loads(profile.tools_json or "[]") if isinstance(parsed_tools, list): tools = [str(t) for t in parsed_tools] + else: + tools_parse_failed = True except (json.JSONDecodeError, TypeError): + tools_parse_failed = True logging.warning( f"[ACP/agent-config] profile {profile.name!r} has invalid " f"tools_json — using fallback tool set" @@ -358,8 +437,12 @@ def get_acp_agent_config( str(c).strip() for c in raw_allowlist if c ] except Exception as error: - # Never fail the agent's startup on a config lookup problem — it would - # leave the developer with a dead agent rather than a degraded one. + if managed_protocol_version is not None: + raise HTTPException( + status_code=503, + detail="Assigned managed-agent policy could not be resolved", + ) from error + # Legacy/developer runtimes retain their conservative fallback behavior. logging.error( f"[ACP/agent-config] config resolution failed, using fallbacks — {error}", exc_info=True, @@ -367,6 +450,41 @@ def get_acp_agent_config( finally: db.close() + transport: Optional[str] = None + if managed_protocol_version is not None: + if managed_protocol_version != MANAGED_PROTOCOL_VERSION: + raise HTTPException(status_code=426, detail="Unsupported managed protocol") + if agent_profile is None: + raise HTTPException( + status_code=503, + detail="No active study agent profile is assigned to this user", + ) + if tools_parse_failed: + raise HTTPException( + status_code=503, detail="Assigned profile has invalid tools" + ) + if ( + not model + or approval_policy not in SUPPORTED_APPROVAL_POLICIES + or max_iterations < 1 + or not set(tools).issubset(CODE4ME2_AGENT_TOOLS) + ): + raise HTTPException( + status_code=503, detail="Assigned profile policy is incomplete" + ) + if framework_version != MANAGED_RUNTIME: + raise HTTPException( + status_code=409, + detail=f"Assigned runtime {framework_version!r} is not managed in this release", + ) + # Managed runtimes always use this server as transport. Provider location + # and credential references stay server-side. + base_url = None + api_key_ref = None + if max_context_tokens is None: + max_context_tokens = FALLBACK_MAX_CONTEXT_TOKENS + transport = "managed_backend" + return JsonResponseWithStatus( status_code=200, content=AcpAgentConfigGetResponse( @@ -382,10 +500,370 @@ def get_acp_agent_config( approval_policy=approval_policy, temperature=temperature, store_agent_content=store_agent_content, + transport=transport, + managed_protocol_version=( + MANAGED_PROTOCOL_VERSION if transport is not None else None + ), ), ) +def _managed_policy(db, user_id: uuid.UUID, profile) -> dict: + """Build the executable v1 policy; malformed study profiles fail closed.""" + if profile.framework_version != MANAGED_RUNTIME: + raise HTTPException( + status_code=409, + detail=f"Assigned runtime {profile.framework_version!r} is not managed in this release", + ) + try: + tools = json.loads(profile.tools_json) + except (TypeError, json.JSONDecodeError) as error: + raise HTTPException(status_code=503, detail="Assigned profile has invalid tools") from error + if ( + not isinstance(tools, list) + or not all(isinstance(item, str) and item.strip() for item in tools) + or not set(tools).issubset(CODE4ME2_AGENT_TOOLS) + ): + raise HTTPException(status_code=503, detail="Assigned profile has invalid tools") + try: + max_iterations = int(profile.max_steps) + except (TypeError, ValueError): + max_iterations = 0 + temperature = profile.temperature + max_context_tokens = profile.max_context_tokens + if isinstance(temperature, bool) or ( + temperature is not None + and ( + not isinstance(temperature, (int, float)) + or not 0.0 <= float(temperature) <= 2.0 + ) + ): + raise HTTPException(status_code=503, detail="Assigned profile temperature is invalid") + if max_context_tokens is None: + max_context_tokens = FALLBACK_MAX_CONTEXT_TOKENS + if ( + isinstance(max_context_tokens, bool) + or not isinstance(max_context_tokens, int) + or max_context_tokens < 1 + ): + raise HTTPException(status_code=503, detail="Assigned profile context limit is invalid") + if ( + not isinstance(profile.model, str) + or not profile.model.strip() + or profile.approval_policy not in SUPPORTED_APPROVAL_POLICIES + or max_iterations < 1 + ): + raise HTTPException(status_code=503, detail="Assigned profile policy is incomplete") + + commands_allowlist = list(FALLBACK_COMMANDS_ALLOWLIST) + user = crud.get_user_by_id(db, user_id) + if user is not None: + config_row = crud.get_config_by_id(db, user.config_id) + if config_row is not None and config_row.config_data: + raw_allowlist = _parse_agent_config_section(config_row.config_data).get( + "commands_allowlist" + ) + if isinstance(raw_allowlist, list): + if not all(isinstance(command, str) and command.strip() for command in raw_allowlist): + raise HTTPException(status_code=503, detail="Assigned command policy is invalid") + commands_allowlist = [command.strip() for command in raw_allowlist] + + return { + "version": MANAGED_PROTOCOL_VERSION, + "transport": "managed_backend", + "agent_profile": profile.name, + "framework_version": profile.framework_version, + "model": profile.model.strip(), + "tools": tools, + "approval_policy": profile.approval_policy, + "temperature": float(temperature) if temperature is not None else None, + "max_iterations": max_iterations, + "max_context_tokens": max_context_tokens, + "commands_allowlist": commands_allowlist, + "store_agent_content": resolve_store_agent_content_for_acp(db, str(user_id)), + } + + +@router.get("/readiness") +def get_participant_readiness( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +) -> dict: + """Resolve and validate the signed-in participant's sticky study arm.""" + db = app.get_db_session() + try: + from agents import registry + + profile = registry.resolve_assignment(db, current_user.user_id) + if profile is None: + raise HTTPException( + status_code=503, + detail="No active study agent profile is assigned to this user", + ) + policy = _managed_policy(db, current_user.user_id, profile) + if profile.api_key_ref and not os.getenv(profile.api_key_ref, "").strip(): + raise HTTPException( + status_code=503, + detail=( + "The assigned model provider is not configured on the study server " + f"(missing {profile.api_key_ref})" + ), + ) + return { + "ready": True, + "managed_protocol_version": MANAGED_PROTOCOL_VERSION, + "runtime": profile.framework_version, + "agent_profile": profile.name, + "policy_version": policy["version"], + } + finally: + db.close() + + +def _scope_ids(scope: AcpServerAuthorization) -> tuple[uuid.UUID, uuid.UUID, uuid.UUID]: + try: + return ( + uuid.UUID(str(scope.user_id)), + uuid.UUID(str(scope.project_id)), + uuid.UUID(str(scope.session_id)), + ) + except (TypeError, ValueError) as error: + raise HTTPException(status_code=401, detail="ACP authorization scope is malformed") from error + + +def _require_managed_task(db, body: ManagedRunRequest, scope: AcpServerAuthorization): + user_id, project_id, _ = _scope_ids(scope) + task = crud.get_agent_task_by_external_run_id(db, body.run_id) + if task is None: + raise HTTPException(status_code=404, detail="Managed run does not exist") + if ( + task.owner_user_id != user_id + or task.owner_project_id != project_id + or task.agent_session_id != body.session_id + or task.source != "code4me2_agent" + ): + raise HTTPException(status_code=409, detail="Managed run belongs to another scope") + return task + + +@router.post("/runs") +def create_managed_run( + body: ManagedRunRequest, + app: App = Depends(App.get_instance), + scope: AcpServerAuthorization = Depends(require_acp_server_scope), +) -> JSONResponse: + """Idempotently create a self-reporting managed runtime task.""" + user_id, project_id, parent_session_id = _scope_ids(scope) + db = app.get_db_session() + try: + existing = crud.get_agent_task_by_external_run_id(db, body.run_id) + if existing is not None: + task = _require_managed_task(db, body, scope) + if not isinstance(task.policy_snapshot, dict): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": task.policy_snapshot}, + status_code=200, + ) + + from agents import registry + + profile = registry.resolve_assignment(db, user_id) + if profile is None: + raise HTTPException( + status_code=503, detail="No active study agent profile is assigned to this user" + ) + policy = _managed_policy(db, user_id, profile) + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + session_id=parent_session_id, + owner_user_id=user_id, + owner_project_id=project_id, + external_run_id=body.run_id, + agent_session_id=body.session_id, + source="code4me2_agent", + status="running", + started_at=datetime.now(timezone.utc), + policy_snapshot=policy, + ) + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": policy}, + status_code=201, + ) + except HTTPException: + raise + except Exception as error: + # A concurrent identical request can race the unique external_run_id. + # Roll back before attempting an ownership-safe idempotent read. + db.rollback() + existing = crud.get_agent_task_by_external_run_id(db, body.run_id) + if existing is not None: + task = _require_managed_task(db, body, scope) + if not isinstance(task.policy_snapshot, dict): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": task.policy_snapshot}, + status_code=200, + ) + logging.error("[ACP/runs] failed to create managed run: %s", error, exc_info=True) + raise HTTPException(status_code=500, detail="Failed to create managed run") from error + finally: + db.close() + + +@router.post("/inference") +async def run_managed_inference( + body: ManagedInferenceRequest, + app: App = Depends(App.get_instance), + scope: AcpServerAuthorization = Depends(require_acp_server_scope), +): + """Relay a managed runtime model call without proxy-generated telemetry.""" + if body.request.get("stream"): + raise HTTPException(status_code=400, detail="Managed protocol v1 requires non-streaming inference") + if "input" in body.request or not isinstance(body.request.get("messages"), list): + raise HTTPException(status_code=400, detail="Managed protocol v1 requires a Chat Completions request") + + _, _, parent_session_id = _scope_ids(scope) + db = app.get_db_session() + try: + task = _require_managed_task(db, body, scope) + policy = task.policy_snapshot + if not _valid_managed_policy_snapshot(policy): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + profile = crud.get_agent_profile(db, task.agent_profile) + if profile is None: + raise HTTPException(status_code=503, detail="Managed run provider configuration is unavailable") + snapshot = { + "task_id": task.task_id, + "agent_profile": task.agent_profile, + "model": policy.get("model"), + "temperature": policy.get("temperature"), + "tools_json": json.dumps(policy.get("tools", [])), + "framework_version": task.framework_version, + "base_url": profile.base_url, + "api_key_ref": profile.api_key_ref, + "content_included": bool(policy.get("store_agent_content", False)), + } + finally: + db.close() + + from agents import inference + + model_request = dict(body.request) + model_request["model"] = snapshot["model"] + _enforce_inference_tool_policy(model_request, policy_tools=policy["tools"]) + if snapshot["temperature"] is None: + # A participant-supplied request may not broaden or change the assigned + # experiment policy. Absence in the snapshot means provider default, + # not permission to provide a client-side override. + model_request.pop("temperature", None) + else: + model_request["temperature"] = snapshot["temperature"] + + return await inference.run_inference( + task_uuid=snapshot["task_id"], + session_uuid=parent_session_id, + openai_body=model_request, + enrichment=None, + agent_profile=snapshot["agent_profile"], + model=snapshot["model"], + temperature=snapshot["temperature"], + base_url=snapshot["base_url"], + api_key_ref=snapshot["api_key_ref"], + framework_version=snapshot["framework_version"], + profile_tools_json=snapshot["tools_json"], + content_included=snapshot["content_included"], + record_observation_events=False, + app=app, + ) + + +def _valid_managed_policy_snapshot(policy: object) -> bool: + if not isinstance(policy, dict): + return False + temperature = policy.get("temperature") + context_limit = policy.get("max_context_tokens") + iterations = policy.get("max_iterations") + tools = policy.get("tools") + return bool( + policy.get("version") == MANAGED_PROTOCOL_VERSION + and isinstance(policy.get("model"), str) + and policy["model"].strip() + and policy.get("approval_policy") in SUPPORTED_APPROVAL_POLICIES + and isinstance(iterations, int) + and not isinstance(iterations, bool) + and iterations >= 1 + and isinstance(context_limit, int) + and not isinstance(context_limit, bool) + and context_limit >= 1 + and ( + temperature is None + or ( + isinstance(temperature, (int, float)) + and not isinstance(temperature, bool) + and 0.0 <= float(temperature) <= 2.0 + ) + ) + and isinstance(tools, list) + and all(isinstance(tool, str) and tool.strip() for tool in tools) + and set(tools).issubset(CODE4ME2_AGENT_TOOLS) + ) + + +def _enforce_inference_tool_policy( + model_request: dict, + *, + policy_tools: list[str], +) -> None: + """Prevent a managed client from broadening its snapshotted tool set.""" + allowed = set(policy_tools) + wildcard_mcp = "mcp__*" in allowed + requested = model_request.get("tools") + if requested is None: + model_request.pop("tool_choice", None) + return + if not isinstance(requested, list): + raise HTTPException(status_code=400, detail="Managed inference tools must be a list") + for definition in requested: + if not isinstance(definition, dict): + raise HTTPException(status_code=400, detail="Managed inference tool definition is invalid") + function = definition.get("function") + name = function.get("name") if isinstance(function, dict) else None + if not isinstance(name, str) or not name.strip(): + raise HTTPException(status_code=400, detail="Managed inference tool definition is invalid") + if name not in allowed and not (wildcard_mcp and name.startswith("mcp__")): + raise HTTPException( + status_code=403, + detail=f"Tool {name!r} is not enabled by the managed run policy", + ) + tool_choice = model_request.get("tool_choice") + if isinstance(tool_choice, dict): + function = tool_choice.get("function") + selected_name = function.get("name") if isinstance(function, dict) else None + if not isinstance(selected_name, str) or not selected_name.strip(): + raise HTTPException(status_code=400, detail="Managed inference tool choice is invalid") + requested_names = { + definition["function"]["name"] + for definition in requested + } + if selected_name not in requested_names: + raise HTTPException( + status_code=403, + detail=f"Tool choice {selected_name!r} is not present in the managed request", + ) + elif tool_choice not in (None, "auto", "none", "required"): + raise HTTPException(status_code=400, detail="Managed inference tool choice is invalid") + if not requested: + model_request.pop("tools", None) + model_request.pop("tool_choice", None) + + def _parse_agent_config_section(config_data: str) -> dict: """Extract the optional ``agent`` section from a user's config blob. diff --git a/src/backend/routers/agent/ingest.py b/src/backend/routers/agent/ingest.py index 7ce1827..0a5dd7e 100644 --- a/src/backend/routers/agent/ingest.py +++ b/src/backend/routers/agent/ingest.py @@ -23,13 +23,13 @@ from datetime import datetime from typing import Any, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field from agents import ingest as ingest_module from agents import lifecycle, registry from App import App -from backend.acp_authorization import AcpSessionAuthorization +from backend.acp_authorization import AcpSessionAuthorization # noqa: TC001 - FastAPI route from backend.Responses import JsonResponseWithStatus from backend.routers.agent.acp_auth import require_acp_scope from backend.routers.agent.consent import resolve_store_agent_content_for_acp @@ -101,9 +101,10 @@ def _resolve_or_create_task( """Find the task backing this run, creating it on the first batch. Returns ``(task, created)``. Raises 403 if a task exists for this run id but - belongs to a different user — the run id is runtime-generated, so an - ownership check is the only thing preventing one agent process from writing - into another user's task. + belongs to a different user or project, and 409 if it belongs to a different + ACP session — the run id is runtime-generated, so ownership checks are the + only thing preventing one agent process from writing into another scope's + task. """ task = crud.get_agent_task_by_external_run_id(db, run.run_id) @@ -125,6 +126,33 @@ def _resolve_or_create_task( status_code=403, detail="Agent run is not authorized for this ACP session.", ) + # Managed runs are bound to one project and one ACP session at creation + # (/api/acp/runs). A second project reusing the same run_id — e.g. two + # IDE windows sharing a user — must not append telemetry to another + # project's task. + if ( + task.owner_project_id is not None + and owner_project_uuid is not None + and task.owner_project_id != owner_project_uuid + ): + logging.warning( + f"[Agent/ingest] 403 — run {run.run_id} owned by another project" + ) + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + if ( + task.agent_session_id is not None + and run.session_id != task.agent_session_id + ): + logging.warning( + f"[Agent/ingest] 409 — run {run.run_id} belongs to another ACP session" + ) + raise HTTPException( + status_code=409, + detail="Agent run belongs to another ACP session.", + ) return task, False profile = registry.resolve_assignment(db, owner_user_uuid) @@ -173,6 +201,18 @@ def ingest_agent_events( raise HTTPException( status_code=400, detail="All events must belong to the request run_id." ) + if body.run.source != "code4me2_agent" or any( + event.source != "code4me2_agent" for event in body.events + ): + raise HTTPException( + status_code=400, + detail="Managed telemetry must use the code4me2_agent source.", + ) + if any(event.session_id != body.run.session_id for event in body.events): + raise HTTPException( + status_code=400, + detail="All events must belong to the request ACP session.", + ) db = app.get_db_session() try: @@ -225,6 +265,7 @@ def ingest_agent_events( @router.get("/runs/{run_id}", summary="Read back a self-reported agent run") def get_agent_run( run_id: str, + session_id: str = Query(..., min_length=1, max_length=500), app: App = Depends(App.get_instance), scope: AcpSessionAuthorization = Depends(require_acp_scope), ) -> JsonResponseWithStatus: @@ -243,6 +284,20 @@ def get_agent_run( status_code=403, detail="Agent run is not authorized for this ACP session.", ) + try: + owner_project_uuid = uuid.UUID(str(scope.project_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid project") + if task.owner_project_id != owner_project_uuid: + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP project.", + ) + if task.agent_session_id != session_id: + raise HTTPException( + status_code=409, + detail="Agent run belongs to another ACP session.", + ) events = crud.get_agent_events_by_task(db, task.task_id) return JsonResponseWithStatus( diff --git a/src/backend/routers/agent/profiles.py b/src/backend/routers/agent/profiles.py index 6714b4a..3e17fac 100644 --- a/src/backend/routers/agent/profiles.py +++ b/src/backend/routers/agent/profiles.py @@ -20,12 +20,14 @@ from __future__ import annotations import json -import uuid +import re +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime from datetime import datetime from typing import Any, Optional +from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from sqlalchemy.exc import IntegrityError from agents.tools import KNOWN_AGENT_TOOLS, tools_for_framework @@ -37,7 +39,7 @@ require_admin, ) from database import crud -from database.db_schemas import AgentProfile, AgentProfileAssignment +from database.db_schemas import AgentProfile, AgentProfileAssignment # noqa: TC001 router = APIRouter() @@ -45,6 +47,7 @@ # no way to launch, so it's rejected at the API boundary rather than failing # later at agent-startup time. SUPPORTED_FRAMEWORKS = ("code4me2-agent", "goose", "codex") +SUPPORTED_APPROVAL_POLICIES = ("auto", "per_step", "suggestion_only") class AgentProfilePayload(BaseModel): @@ -64,6 +67,34 @@ class AgentProfilePayload(BaseModel): temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) max_context_tokens: Optional[int] = Field(default=None, ge=1) + @field_validator("name", "model") + @classmethod + def validate_required_text(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("value must not be blank") + return normalized + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: Optional[str]) -> Optional[str]: + if value is None or not value.strip(): + return None + candidate = value.strip().rstrip("/") + parsed = urlparse(candidate) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError( + "base_url must be an HTTP(S) endpoint without credentials, query, or fragment" + ) + return candidate + @field_validator("tools_json") @classmethod def validate_tools_json(cls, value: str) -> str: @@ -88,6 +119,28 @@ def validate_framework(cls, value: str) -> str: ) return normalized + @field_validator("approval_policy") + @classmethod + def validate_approval_policy(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in SUPPORTED_APPROVAL_POLICIES: + raise ValueError( + "approval_policy must be one of " + + ", ".join(SUPPORTED_APPROVAL_POLICIES) + ) + return normalized + + @model_validator(mode="after") + def validate_tools_for_framework(self): + selected = set(json.loads(self.tools_json)) + unknown = selected - set(tools_for_framework(self.framework_version)) + if unknown: + raise ValueError( + f"tools_json contains tools unsupported by {self.framework_version}: " + + ", ".join(sorted(unknown)) + ) + return self + @field_validator("api_key_ref") @classmethod def validate_api_key_ref(cls, value: Optional[str]) -> Optional[str]: @@ -103,7 +156,7 @@ def validate_api_key_ref(cls, value: Optional[str]) -> Optional[str]: candidate = value.strip() if not candidate: return None - if len(candidate) > 128 or any(ch in candidate for ch in "-. /:"): + if not re.fullmatch(r"[A-Z_][A-Z0-9_]{0,127}", candidate): raise ValueError( "api_key_ref must be the NAME of an environment variable " "(e.g. OPENAI_API_KEY), not an API key value" diff --git a/src/database/crud.py b/src/database/crud.py index b0aad8c..d8b9a2c 100644 --- a/src/database/crud.py +++ b/src/database/crud.py @@ -1473,6 +1473,7 @@ def create_agent_task( agent_session_id: Optional[str] = None, status: str = "pending", started_at: Optional[datetime] = None, + policy_snapshot: Optional[dict] = None, ) -> db_schemas.AgentTask: task = db_schemas.AgentTask( task_id=task_id or uuid.uuid4(), @@ -1490,6 +1491,7 @@ def create_agent_task( external_run_id=external_run_id, agent_session_id=agent_session_id, started_at=started_at, + policy_snapshot=policy_snapshot, task_description=task_description, ) db.add(task) diff --git a/src/database/db_schemas.py b/src/database/db_schemas.py index 7e4c00e..36f2ba7 100644 --- a/src/database/db_schemas.py +++ b/src/database/db_schemas.py @@ -31,7 +31,7 @@ String, Text, ) -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import relationship from .db import Base @@ -866,6 +866,8 @@ class AgentTask(Base): # the profile, but left nullable so the proxy can backfill the concrete # version string it observes ("goose 1.x") on the first inference call. framework_version = Column(String, nullable=True) + # Complete, versioned managed-runtime policy. Nullable for legacy tasks. + policy_snapshot = Column(JSONB, nullable=True) status = Column(String, nullable=False) # pending | running | done | failed created_at = Column(DateTime, default=datetime.now) started_at = Column(DateTime(timezone=True), nullable=True) diff --git a/src/database/migration/migration_manager.py b/src/database/migration/migration_manager.py index 5495918..3922c7d 100644 --- a/src/database/migration/migration_manager.py +++ b/src/database/migration/migration_manager.py @@ -20,13 +20,26 @@ from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from dotenv import load_dotenv from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url # Set up paths current_dir = Path(__file__).parent # src/database/migration project_root = current_dir.parent.parent.parent # project root src_dir = project_root / "src" sys.path.insert(0, str(src_dir)) +# Match src/main.py: operator-supplied environment variables win, while a +# repository/deployment .env fills values that Docker Compose does not export. +load_dotenv(project_root / ".env", override=False) + + +def _database_url_for_log(value: str) -> str: + try: + return make_url(value).render_as_string(hide_password=True) + except Exception: + return "" class MigrationManager: @@ -190,77 +203,90 @@ def ensure_initialized(self) -> bool: return True - def init_migrations(self) -> None: + def init_migrations(self) -> bool: """Initialize migrations with hybrid approach.""" db_type = "test database" if self.use_test_db else "main database" print(f"Initializing migration system for {db_type}...") - print(f"Database URL: {self.get_database_url()}") + print(f"Database URL: {_database_url_for_log(self.get_database_url())}") if self.ensure_initialized(): print("Migration system ready!") print("Use 'create' to add new migrations, 'migrate' to apply them") + return True else: print("Failed to initialize migration system") + return False - def create_migration(self, message: str) -> None: + def create_migration(self, message: str) -> bool: """Create a new migration.""" print(f"Creating migration: {message}") # Ensure system is initialized first if not self.ensure_initialized(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.revision(self.alembic_cfg, message=message, autogenerate=True) print("Migration created") + return True except Exception as e: print(f"Error: {e}") + return False - def migrate(self) -> None: + def migrate(self) -> bool: """Apply all migrations.""" print("Applying migrations...") # Ensure system is initialized first if not self.ensure_initialized(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.upgrade(self.alembic_cfg, "head") + if not self.is_at_expected_head(): + print("Migration command completed, but the database is not at the expected Alembic head") + return False print("Migrations applied") + return True except Exception as e: print(f"Error: {e}") + return False - def current(self) -> None: + def current(self) -> bool: """Show current revision.""" if not self.check_database_connection(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.current(self.alembic_cfg) + return True except Exception as e: print(f"Error: {e}") + return False - def history(self) -> None: + def history(self) -> bool: """Show migration history.""" try: command.history(self.alembic_cfg) + return True except Exception as e: print(f"Error: {e}") + return False - def status(self) -> None: + def status(self) -> bool: """Show detailed status of database and migrations.""" db_type = "Test Database" if self.use_test_db else "Main Database" print(f"Migration System Status - {db_type}") print("=" * 50) - print(f"Database URL: {self.get_database_url()}") + print(f"Database URL: {_database_url_for_log(self.get_database_url())}") # Check database connection if not self.check_database_connection(): print("Database: Not connected") - return + return False print("Database: Connected") @@ -297,18 +323,40 @@ def status(self) -> None: ) version = result.scalar() print(f"Current version: {version}") + if not self.is_at_expected_head(): + print("Migration readiness: Not at the expected head") + return False + print("Migration readiness: At expected head") except Exception: print("Could not get current version") + return False else: print("Migration tracking: Not set up") + return False + return True - def reset(self) -> None: + def is_at_expected_head(self) -> bool: + """Require the deployed database revision set to equal the code's heads.""" + try: + expected = set(ScriptDirectory.from_config(self.alembic_cfg).get_heads()) + engine = create_engine(self.get_database_url()) + with engine.connect() as conn: + applied = { + str(row[0]) + for row in conn.execute(text("SELECT version_num FROM alembic_version")) + } + return bool(expected) and applied == expected + except Exception as error: + print(f"Could not verify migration head: {error}") + return False + + def reset(self) -> bool: """Reset database and reinitialize.""" db_type = "test database" if self.use_test_db else "main database" print(f"Resetting {db_type}...") if not self.check_database_connection(): - return + return False try: # Drop and recreate public schema @@ -321,13 +369,14 @@ def reset(self) -> None: print("Database reset") # Reinitialize - self.init_migrations() + return self.init_migrations() except Exception as e: print(f"Error during reset: {e}") + return False -def main(): +def main() -> int: """CLI interface.""" parser = argparse.ArgumentParser(description="Hybrid Migration Manager") parser.add_argument( @@ -354,25 +403,26 @@ def main(): if not args.command: parser.print_help() - return + return 0 manager = MigrationManager(use_test_db=args.test) if args.command == "init": - manager.init_migrations() + succeeded = manager.init_migrations() elif args.command == "status": - manager.status() + succeeded = manager.status() elif args.command == "create": - manager.create_migration(args.message) + succeeded = manager.create_migration(args.message) elif args.command == "migrate": - manager.migrate() + succeeded = manager.migrate() elif args.command == "current": - manager.current() + succeeded = manager.current() elif args.command == "history": - manager.history() + succeeded = manager.history() elif args.command == "reset": - manager.reset() + succeeded = manager.reset() + return 0 if succeeded else 1 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py b/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py new file mode 100644 index 0000000..eed7172 --- /dev/null +++ b/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py @@ -0,0 +1,27 @@ +"""add managed agent policy snapshot + +Revision ID: f3c4d5e6a7b9 +Revises: e2b3c4d5f6a8 +Create Date: 2026-09-10 +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "f3c4d5e6a7b9" +down_revision = "e2b3c4d5f6a8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "agent_task", + sa.Column("policy_snapshot", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + schema="public", + ) + + +def downgrade() -> None: + op.drop_column("agent_task", "policy_snapshot", schema="public") diff --git a/tests/backend_tests/test_managed_acp.py b/tests/backend_tests/test_managed_acp.py new file mode 100644 index 0000000..30f5511 --- /dev/null +++ b/tests/backend_tests/test_managed_acp.py @@ -0,0 +1,486 @@ +import asyncio +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from backend.acp_authorization import ( + AcpAuthorizationService, + AcpServerAuthorization, + AcpSessionAuthorization, +) +from backend.routers.acp import ( + EXPECTED_SCHEMA_REVISION, + ManagedInferenceRequest, + ManagedRunRequest, + _managed_policy, + create_managed_run, + get_acp_capabilities, + get_participant_readiness, + run_managed_inference, +) +from backend.routers.agent.ingest import get_agent_run +from backend.routers.agent.profiles import AgentProfilePayload +from backend.routers.analytics.auth_utils import AuthenticatedUser + + +class MemoryTokenStore: + def __init__(self): + self.data = {} + + def get(self, token_type, token, reset_exp=False): + return self.data.get((token_type, token)) + + def set(self, token_type, token, info, force_reset_exp=False): + self.data[(token_type, token)] = dict(info) + + def consume(self, token_type, token): + return self.data.pop((token_type, token), None) + + def discard(self, token_type, token): + self.data.pop((token_type, token), None) + + +def _authorized_store(): + store = MemoryTokenStore() + store.set("auth_token", "auth", {"user_id": "user"}) + store.set("user_token", "user", {"session_token": "parent"}) + store.set("session_token", "parent", {"project_tokens": ["project"]}) + store.set("project_token", "project", {"session_tokens": ["parent"]}) + return store + + +def _scope(): + return AcpServerAuthorization( + acp_token="acp", + user_id=str(uuid.uuid4()), + session_id=str(uuid.uuid4()), + project_id=str(uuid.uuid4()), + project_info={}, + workspace="/workspace", + ) + + +def test_managed_grants_are_isolated_per_launch_and_accept_windows_paths(): + store = _authorized_store() + tokens = iter(["grant-one", "grant-two", "acp-token"]) + service = AcpAuthorizationService(store, token_factory=lambda: next(tokens)) + + first = service.prepare_grant( + auth_token="auth", + project_id="project", + workspace=r"C:\study\project", + launch_id="launch-one", + path_format="windows", + managed_protocol_version="1", + ) + second = service.prepare_grant( + auth_token="auth", + project_id="project", + workspace=r"C:\study\project", + launch_id="launch-two", + path_format="windows", + managed_protocol_version="1", + ) + retried = service.prepare_grant( + auth_token="auth", + project_id="project", + workspace=r"C:\study\project", + launch_id="launch-one", + path_format="windows", + managed_protocol_version="1", + ) + + assert first.workspace == r"C:\study\project" + assert retried.grant == first.grant + assert store.get("acp_grant", first.grant) is not None + assert store.get("acp_grant", second.grant) is not None + assert store.get("acp_grant", retried.grant) is not None + session = service.exchange_grant(second.grant) + assert session.acp_token == "acp-token" + + +def test_capabilities_report_schema_readiness(): + db = MagicMock() + db.execute.return_value.scalar.return_value = EXPECTED_SCHEMA_REVISION + app = MagicMock() + app.get_db_session.return_value = db + + result = get_acp_capabilities(app) + + assert result["schema_ready"] is True + assert result["managed_protocol_versions"] == ["1"] + assert result["supported_runtimes"] == ["code4me2-agent"] + db.close.assert_called_once() + + +def test_readiness_rejects_an_uncertified_assigned_runtime(): + user_id = uuid.uuid4() + profile = SimpleNamespace(framework_version="goose", name="goose-arm") + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + user = AuthenticatedUser(user_id=user_id, is_admin=False, email="p@example.com", name="P") + + with patch("agents.registry.resolve_assignment", return_value=profile): + with pytest.raises(Exception) as error: + get_participant_readiness(user, app) + + assert error.value.status_code == 409 + assert "not managed" in error.value.detail + db.close.assert_called_once() + + +def test_readiness_rejects_missing_server_side_provider_credential(monkeypatch): + user_id = uuid.uuid4() + profile = SimpleNamespace( + framework_version="code4me2-agent", + name="managed-arm", + model="managed-model", + tools_json="[]", + approval_policy="auto", + max_steps=3, + max_context_tokens=1000, + temperature=0.2, + api_key_ref="STUDY_PROVIDER_KEY", + ) + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + user = AuthenticatedUser(user_id=user_id, is_admin=False, email="p@example.com", name="P") + monkeypatch.delenv("STUDY_PROVIDER_KEY", raising=False) + + with patch("agents.registry.resolve_assignment", return_value=profile), patch( + "backend.routers.acp.crud.get_user_by_id", return_value=None + ), patch("backend.routers.acp.resolve_store_agent_content_for_acp", return_value=False): + with pytest.raises(Exception) as error: + get_participant_readiness(user, app) + + assert error.value.status_code == 503 + assert "STUDY_PROVIDER_KEY" in error.value.detail + db.close.assert_called_once() + + +def test_managed_policy_applies_the_assigned_command_allowlist(): + user_id = uuid.uuid4() + profile = SimpleNamespace( + framework_version="code4me2-agent", + name="managed-arm", + model="managed-model", + tools_json='["run_command"]', + approval_policy="per_step", + max_steps=3, + max_context_tokens=1000, + temperature=0.2, + ) + user = SimpleNamespace(config_id=uuid.uuid4()) + config = SimpleNamespace( + config_data='{"agent":{"commands_allowlist":["git", "rg"]}}' + ) + + with patch("backend.routers.acp.crud.get_user_by_id", return_value=user), patch( + "backend.routers.acp.crud.get_config_by_id", return_value=config + ), patch( + "backend.routers.acp.resolve_store_agent_content_for_acp", return_value=False + ): + policy = _managed_policy(MagicMock(), user_id, profile) + + assert policy["commands_allowlist"] == ["git", "rg"] + + +def test_profile_validation_rejects_tools_from_another_runtime(): + with pytest.raises(ValueError, match="unsupported by code4me2-agent"): + AgentProfilePayload( + name="invalid-arm", + model="model", + framework_version="code4me2-agent", + tools_json='["shell"]', + approval_policy="auto", + max_steps=3, + ) + + +def test_profile_validation_rejects_unknown_approval_policy(): + with pytest.raises(ValueError, match="approval_policy"): + AgentProfilePayload( + name="invalid-arm", + model="model", + framework_version="code4me2-agent", + tools_json='["read_file"]', + approval_policy="sometimes", + max_steps=3, + ) + + +def test_profile_validation_rejects_provider_secrets_and_unsafe_urls(): + common = { + "name": "managed-arm", + "model": "model", + "framework_version": "code4me2-agent", + "tools_json": "[]", + "approval_policy": "auto", + "max_steps": 3, + } + with pytest.raises(ValueError, match="NAME of an environment variable"): + AgentProfilePayload(**common, api_key_ref="gsk_actualSecretMaterial123456") + with pytest.raises(ValueError, match="base_url"): + AgentProfilePayload(**common, base_url="https://user:secret@provider.example/v1") + + valid = AgentProfilePayload( + **common, + api_key_ref="STUDY_PROVIDER_KEY", + base_url="https://provider.example/v1/", + ) + assert valid.api_key_ref == "STUDY_PROVIDER_KEY" + assert valid.base_url == "https://provider.example/v1" + + +def test_managed_run_is_idempotent_only_for_the_same_scope(): + scope = _scope() + task = SimpleNamespace( + task_id=uuid.uuid4(), + external_run_id="run-1", + owner_user_id=uuid.UUID(scope.user_id), + owner_project_id=uuid.UUID(scope.project_id), + agent_session_id="acp-session-1", + source="code4me2_agent", + policy_snapshot={"version": "1"}, + ) + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + + with patch("backend.routers.acp.crud.get_agent_task_by_external_run_id", return_value=task): + response = create_managed_run( + ManagedRunRequest(run_id="run-1", session_id="acp-session-1"), app, scope + ) + + assert response.status_code == 200 + assert b'"run_id":"run-1"' in response.body + + task.owner_project_id = uuid.uuid4() + with patch("backend.routers.acp.crud.get_agent_task_by_external_run_id", return_value=task): + with pytest.raises(Exception) as error: + create_managed_run( + ManagedRunRequest(run_id="run-1", session_id="acp-session-1"), app, scope + ) + assert error.value.status_code == 409 + + +def test_managed_inference_disables_proxy_observation_events(): + scope = _scope() + task = SimpleNamespace( + task_id=uuid.uuid4(), + owner_user_id=uuid.UUID(scope.user_id), + owner_project_id=uuid.UUID(scope.project_id), + agent_session_id="acp-session-1", + source="code4me2_agent", + policy_snapshot={ + "version": "1", + "model": "managed-model", + "tools": ["read_file"], + "approval_policy": "auto", + "max_iterations": 3, + "max_context_tokens": 1000, + "temperature": None, + "store_agent_content": False, + }, + agent_profile="managed-profile", + framework_version="code4me2-agent", + ) + profile = SimpleNamespace(base_url="https://provider.example/v1", api_key_ref="KEY") + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + forwarded = MagicMock() + + with patch( + "backend.routers.acp.crud.get_agent_task_by_external_run_id", return_value=task + ), patch("backend.routers.acp.crud.get_agent_profile", return_value=profile), patch( + "agents.inference.run_inference", new=AsyncMock(return_value=forwarded) + ) as run_inference: + result = asyncio.run( + run_managed_inference( + ManagedInferenceRequest( + run_id="run-1", + session_id="acp-session-1", + request={ + "model": "participant-override", + "temperature": 1.9, + "messages": [{"role": "user", "content": "hello"}], + }, + ), + app, + scope, + ) + ) + + assert result is forwarded + assert run_inference.await_args.kwargs["record_observation_events"] is False + assert run_inference.await_args.kwargs["model"] == "managed-model" + assert run_inference.await_args.kwargs["openai_body"]["model"] == "managed-model" + assert "temperature" not in run_inference.await_args.kwargs["openai_body"] + + +def test_managed_inference_rejects_tools_outside_snapshot(): + scope = _scope() + task = SimpleNamespace( + task_id=uuid.uuid4(), + owner_user_id=uuid.UUID(scope.user_id), + owner_project_id=uuid.UUID(scope.project_id), + agent_session_id="acp-session-1", + source="code4me2_agent", + policy_snapshot={ + "version": "1", + "model": "managed-model", + "tools": ["read_file"], + "approval_policy": "auto", + "max_iterations": 3, + "max_context_tokens": 1000, + "temperature": 0.2, + }, + agent_profile="managed-profile", + framework_version="code4me2-agent", + ) + profile = SimpleNamespace(base_url="https://provider.example/v1", api_key_ref="KEY") + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + + with patch( + "backend.routers.acp.crud.get_agent_task_by_external_run_id", return_value=task + ), patch("backend.routers.acp.crud.get_agent_profile", return_value=profile): + with pytest.raises(Exception) as error: + asyncio.run( + run_managed_inference( + ManagedInferenceRequest( + run_id="run-1", + session_id="acp-session-1", + request={ + "messages": [{"role": "user", "content": "hello"}], + "tools": [ + { + "type": "function", + "function": {"name": "write_file", "parameters": {}}, + } + ], + }, + ), + app, + scope, + ) + ) + assert error.value.status_code == 403 + + +def test_managed_inference_rejects_tool_choice_not_in_request(): + scope = _scope() + task = SimpleNamespace( + task_id=uuid.uuid4(), + owner_user_id=uuid.UUID(scope.user_id), + owner_project_id=uuid.UUID(scope.project_id), + agent_session_id="acp-session-1", + source="code4me2_agent", + policy_snapshot={ + "version": "1", + "model": "managed-model", + "tools": ["read_file", "write_file"], + "approval_policy": "auto", + "max_iterations": 3, + "max_context_tokens": 1000, + "temperature": 0.2, + }, + agent_profile="managed-profile", + framework_version="code4me2-agent", + ) + profile = SimpleNamespace(base_url="https://provider.example/v1", api_key_ref="KEY") + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + + with patch( + "backend.routers.acp.crud.get_agent_task_by_external_run_id", return_value=task + ), patch("backend.routers.acp.crud.get_agent_profile", return_value=profile): + with pytest.raises(Exception) as error: + asyncio.run( + run_managed_inference( + ManagedInferenceRequest( + run_id="run-1", + session_id="acp-session-1", + request={ + "messages": [{"role": "user", "content": "hello"}], + "tools": [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {}}, + } + ], + "tool_choice": { + "type": "function", + "function": {"name": "write_file"}, + }, + }, + ), + app, + scope, + ) + ) + assert error.value.status_code == 403 + + +def test_managed_inference_rejects_non_chat_completions_payload(): + with pytest.raises(Exception) as error: + asyncio.run( + run_managed_inference( + ManagedInferenceRequest( + run_id="run-1", + session_id="acp-session-1", + request={"input": [{"role": "user", "content": "hello"}]}, + ), + MagicMock(), + _scope(), + ) + ) + assert error.value.status_code == 400 + + +def test_run_readback_is_scoped_to_project_and_acp_session(): + user_id = uuid.uuid4() + project_id = uuid.uuid4() + task = SimpleNamespace( + owner_user_id=user_id, + owner_project_id=project_id, + agent_session_id="acp-session-1", + ) + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + wrong_project_scope = AcpSessionAuthorization( + acp_token="token", + user_id=str(user_id), + project_id=str(uuid.uuid4()), + workspace="/workspace", + ) + + with patch( + "backend.routers.agent.ingest.crud.get_agent_task_by_external_run_id", + return_value=task, + ): + with pytest.raises(Exception) as error: + get_agent_run("run-1", "acp-session-1", app, wrong_project_scope) + assert error.value.status_code == 403 + + right_project_scope = AcpSessionAuthorization( + acp_token="token", + user_id=str(user_id), + project_id=str(project_id), + workspace="/workspace", + ) + with patch( + "backend.routers.agent.ingest.crud.get_agent_task_by_external_run_id", + return_value=task, + ): + with pytest.raises(Exception) as error: + get_agent_run("run-1", "another-session", app, right_project_scope) + assert error.value.status_code == 409 diff --git a/tests/database_tests/test_migration_cli_exit.py b/tests/database_tests/test_migration_cli_exit.py new file mode 100644 index 0000000..b6a9904 --- /dev/null +++ b/tests/database_tests/test_migration_cli_exit.py @@ -0,0 +1,29 @@ +from unittest.mock import MagicMock, patch + +from database.migration import migration_manager + + +def test_migration_cli_returns_nonzero_when_upgrade_fails(): + with patch.object(migration_manager.sys, "argv", ["migration_manager.py", "migrate"]), patch.object( + migration_manager.MigrationManager, "migrate", return_value=False + ): + assert migration_manager.main() == 1 + + +def test_migration_cli_returns_zero_when_upgrade_succeeds(): + with patch.object(migration_manager.sys, "argv", ["migration_manager.py", "migrate"]), patch.object( + migration_manager.MigrationManager, "migrate", return_value=True + ): + assert migration_manager.main() == 0 + + +def test_migrate_fails_when_database_did_not_reach_expected_head(): + manager = object.__new__(migration_manager.MigrationManager) + manager.alembic_cfg = MagicMock() + manager.ensure_initialized = lambda: True + manager.get_database_url = lambda: "postgresql://unused" + manager.is_at_expected_head = lambda: False + + with patch.object(migration_manager.command, "upgrade") as upgrade: + assert manager.migrate() is False + upgrade.assert_called_once() From af06212de7ccae4937a17f2511456c2c444a6340 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Fri, 11 Sep 2026 12:36:20 +0300 Subject: [PATCH 009/115] feat: managed agent runtime with policy enforcement - AgentConfig managed_mode/allowed_tools/approval, frozenset allowlist with mcp__* wildcard - ToolRegistry approval policies (auto/per_step/suggestion_only), definitions filtering, denied-continues loop - acp_runtime managed sessions via plugin bridge, apply_config rebuild; runtime_auth managed inference transport - Backend relay priority managed_backend>code4me_backend>openai, dual tools/allowed_tools compat - Add policy enforcement and CLI tests --- src/code4me2_agent/acp_runtime.py | 257 ++++++++++++--- src/code4me2_agent/adapters.py | 142 +++++++-- src/code4me2_agent/cli.py | 101 +++++- src/code4me2_agent/command_tools.py | 23 ++ src/code4me2_agent/config.py | 105 ++++++- src/code4me2_agent/echo.py | 26 +- src/code4me2_agent/mcp_tools.py | 6 +- src/code4me2_agent/runtime_auth.py | 236 +++++++++++++- src/code4me2_agent/telemetry.py | 73 ++++- tests/code4me2_agent/__init__.py | 1 - tests/code4me2_agent/test_cli.py | 18 ++ .../code4me2_agent/test_policy_enforcement.py | 292 ++++++++++++++++++ 12 files changed, 1187 insertions(+), 93 deletions(-) delete mode 100644 tests/code4me2_agent/__init__.py create mode 100644 tests/code4me2_agent/test_cli.py create mode 100644 tests/code4me2_agent/test_policy_enforcement.py diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py index 4ed3017..efd09b8 100644 --- a/src/code4me2_agent/acp_runtime.py +++ b/src/code4me2_agent/acp_runtime.py @@ -2,6 +2,7 @@ import asyncio import logging +import platform from dataclasses import dataclass, field, replace from pathlib import Path from threading import Event @@ -11,11 +12,15 @@ from code4me2_agent.acp_updates import AcpUpdateBuilder from code4me2_agent.acp_utils import capability_value from code4me2_agent.async_bridge import EventLoopAsyncRunner -from code4me2_agent.command_tools import build_acp_command_backend +from code4me2_agent.command_tools import available_commands, build_acp_command_backend from code4me2_agent.echo import EchoAgentCore from code4me2_agent.file_tools import build_acp_file_system_backend from code4me2_agent.mcp_tools import StdioMcpToolBroker, serialize_mcp_servers -from code4me2_agent.runtime_auth import AcpAuthorizationFailure, AcpBackendAuthorization +from code4me2_agent.runtime_auth import ( + AcpAuthorizationFailure, + AcpBackendAuthorization, + ManagedBridgeAuthorization, +) logger = logging.getLogger(__name__) @@ -127,6 +132,7 @@ class AgentSession: cancel_event: Event = field(default_factory=Event) prompt_lock: asyncio.Lock = field(default_factory=asyncio.Lock) active_prompt_task: asyncio.Task[Any] | None = None + authorization: Any | None = None class AcpSessionEventSink: @@ -206,6 +212,49 @@ def thought(self, event: object) -> None: }, ) + def request_approval(self, tool_call: object, arguments: dict[str, Any]) -> bool: + """Synchronously bridge a worker-thread tool decision to ACP/JetBrains.""" + name = str(getattr(tool_call, "name", "tool")) + tool_call_id = str(getattr(tool_call, "tool_call_id", "")) + metadata = { + "run_command": ("Execute command", "execute"), + "create_file": ("Create file", "edit"), + "write_file": ("Write file", "edit"), + "replace_text": ("Replace text", "edit"), + "read_file": ("Read file", "read"), + "list_files": ("List files", "search"), + "search_files": ("Search files", "search"), + }.get(name, (f"Call {name}", "other")) + permission = self._updates.permission_request( + session_id=self._session_id, + tool_call_id=tool_call_id, + title=metadata[0], + kind=metadata[1], + ) + try: + response = self._async_runner.run( + self._conn.request_permission( + session_id=permission.session_id, + tool_call=permission.tool_call, + options=permission.options, + ) + ) + except Exception: # noqa: BLE001 + logger.exception("ACP permission request failed for tool %s", name) + return False + outcome = getattr(response, "outcome", None) + if isinstance(response, dict): + outcome = response.get("outcome") + if isinstance(outcome, dict): + return ( + outcome.get("outcome") == "selected" + and outcome.get("optionId", outcome.get("option_id")) == "allow_once" + ) + return ( + getattr(outcome, "outcome", None) == "selected" + and getattr(outcome, "option_id", None) == "allow_once" + ) + def _send_update(self, *, event: ToolCallEvent, update: object) -> None: try: self._async_runner.run( @@ -362,6 +411,7 @@ def on_connect(self, conn: Client) -> None: self._sessions = {} self._authorization = ( authorization + or (ManagedBridgeAuthorization() if config.managed_mode else None) or AcpBackendAuthorization.from_environment( workspace_root=config.workspace_root, ) @@ -378,6 +428,7 @@ def _build_session( session_config: AgentConfig, session_id: str, mcp_tools: StdioMcpToolBroker | None, + session_authorization: Any | None = None, ) -> AgentSession: async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) event_sink = AcpSessionEventSink( @@ -408,10 +459,11 @@ def _build_session( session_id=session_id, core=core, mcp_tools=mcp_tools, + authorization=session_authorization or self._authorization, ) def _load_persisted_memory(self, session: AgentSession) -> None: - request_json = getattr(self._authorization, "authorized_json_request", None) + request_json = getattr(session.authorization, "authorized_json_request", None) if not callable(request_json): return try: @@ -431,7 +483,7 @@ def _save_persisted_memory(self, session: AgentSession) -> None: snapshot = session.core.session_memory_snapshot() if snapshot is None: return - request_json = getattr(self._authorization, "authorized_json_request", None) + request_json = getattr(session.authorization, "authorized_json_request", None) if not callable(request_json): return try: @@ -444,7 +496,7 @@ def _save_persisted_memory(self, session: AgentSession) -> None: logger.info("Could not save persisted ACP memory: %s", exc) def _session_config( - self, *, workspace_root: Path, session_id: str + self, *, workspace_root: Path, session_id: str, session_authorization: Any ) -> AgentConfig: # Build from self._current_config, NOT the module-level `config` # closure: _apply_server_config writes the assigned agent profile @@ -454,14 +506,14 @@ def _session_config( # assignment would appear to work (it's logged) but never actually # take effect on a new session. base_config = self._current_config - telemetry_headers = self._authorization.telemetry_headers() + telemetry_headers = session_authorization.telemetry_headers() provider_auth_headers = ( telemetry_headers if base_config.adapter.provider.kind == "code4me_backend" else base_config.adapter.provider.auth_headers ) upload_config = base_config.upload - backend_url = getattr(self._authorization, "backend_url", None) + backend_url = getattr(session_authorization, "backend_url", None) if backend_url and upload_config.ingest_url is None: # Self-reported telemetry converges on the same agent_event # table the inference relay writes to (merge decision 2). @@ -474,7 +526,11 @@ def _session_config( base_config, workspace_root=workspace_root, session_id=session_id, - upload=replace(upload_config, auth_headers=telemetry_headers), + upload=replace( + upload_config, + auth_headers=telemetry_headers, + auth_headers_provider=session_authorization.telemetry_headers, + ), adapter=replace( base_config.adapter, provider=replace( @@ -482,6 +538,11 @@ def _session_config( auth_headers=provider_auth_headers, ), ), + managed_request=( + session_authorization.managed_inference + if base_config.managed_mode + else base_config.managed_request + ), ) async def _create_or_resume_session( @@ -492,18 +553,75 @@ async def _create_or_resume_session( event_type: str, mcp_servers: list[Any] | None, ) -> AgentSession: - await self._require_authenticated() workspace_root = _resolve_session_cwd(cwd) - if workspace_root != self._authorized_workspace: - raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + session_authorization = self._authorization + if config.managed_mode: + session_authorization = ManagedBridgeAuthorization() + try: + await asyncio.to_thread(session_authorization.prepare_workspace, workspace_root) + scope = await asyncio.to_thread(session_authorization.authenticate) + except AcpAuthorizationFailure: + raise RequestError.auth_required({"reason": "authorization_rejected"}) from None + if _resolve_session_cwd(scope.workspace) != workspace_root: + raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + else: + await self._require_authenticated() + if workspace_root != self._authorized_workspace: + raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + base_config = self._current_config + server_config = session_authorization.server_agent_config + if config.managed_mode: + # Managed runs never fall back to local defaults: a missing + # profile, an unsupported runtime, or a malformed policy must + # fail before study work starts. + if server_config is None or not server_config.has_overrides: + raise RequestError.invalid_params( + {"reason": "managed_config_unavailable"} + ) + if ( + server_config.framework_version + and server_config.framework_version + not in {"code4me2-agent", "code4me-agent"} + ): + raise RequestError.invalid_params( + {"reason": "unsupported_managed_runtime", "runtime": server_config.framework_version} + ) + base_config = replace(base_config, workspace_root=workspace_root).with_server_overrides( + server_config, + backend_url=getattr(session_authorization, "backend_url", None), + ) + elif server_config is not None: + base_config = replace(base_config, workspace_root=workspace_root).with_server_overrides( + server_config, + backend_url=getattr(session_authorization, "backend_url", None), + ) + previous_config = self._current_config + self._current_config = base_config session_config = self._session_config( workspace_root=workspace_root, session_id=session_id, + session_authorization=session_authorization, + ) + self._current_config = previous_config + requested_mcp_servers = list(mcp_servers or []) + mcp_allowed_by_policy = ( + not config.managed_mode + or ( + base_config.approval_policy != "suggestion_only" + and base_config.allowed_tools is not None + and any(name.startswith("mcp__") for name in base_config.allowed_tools) + ) ) + enabled_mcp_servers = requested_mcp_servers if mcp_allowed_by_policy else [] + if requested_mcp_servers and not mcp_allowed_by_policy: + logger.info( + "Assigned policy disabled %d client-supplied MCP server(s) before launch.", + len(requested_mcp_servers), + ) try: mcp_tools = await asyncio.to_thread( StdioMcpToolBroker.open, - list(mcp_servers or []), + enabled_mcp_servers, cwd=workspace_root, ) except (RuntimeError, TimeoutError, ValueError) as exc: @@ -522,6 +640,7 @@ async def _create_or_resume_session( session_config=session_config, session_id=session_id, mcp_tools=mcp_tools, + session_authorization=session_authorization, ) except BaseException: if mcp_tools is not None: @@ -544,10 +663,16 @@ async def _create_or_resume_session( request_id=uuid4().hex, payload={ "cwd": session_config.workspace_root.as_posix(), + "runtime_os": platform.system(), + "runtime_architecture": platform.machine(), + "available_commands": available_commands( + session_config.commands.allowlisted_commands + ), "client_capabilities": _normalize_capabilities( self._client_capabilities ), - "mcp_servers_requested": serialize_mcp_servers(mcp_servers), + "mcp_servers_requested": serialize_mcp_servers(requested_mcp_servers), + "mcp_servers_policy_enabled": bool(enabled_mcp_servers), "mcp_backend": "broker" if session.mcp_tools is not None else "none", @@ -610,8 +735,9 @@ async def initialize( self.command_tools = self._bootstrap_core.command_tools initialize_auth_succeeded = False try: - await self._authenticate_if_needed() - initialize_auth_succeeded = True + if not getattr(self._authorization, "needs_workspace", False): + await self._authenticate_if_needed() + initialize_auth_succeeded = True except RequestError: # Some ACP clients never invoke the custom auth handshake for local agents. # Try once during initialize, but keep later session/prompt calls fail-closed. @@ -680,6 +806,9 @@ async def authenticate( ) -> AuthenticateResponse: if method_id != "code4me-plugin-session": raise RequestError.auth_required({"reason": "unsupported_auth_method"}) + if config.managed_mode: + # Workspace-scoped authentication completes at session/new. + return AuthenticateResponse() await self._authenticate_if_needed() return AuthenticateResponse() @@ -776,31 +905,15 @@ def _apply_server_config(self) -> None: ) def _apply_config_to_all_cores(self, new_config: Any) -> None: - from code4me2_agent.command_tools import WorkspaceCommandTools - - self._bootstrap_core._config = new_config - self._bootstrap_core._adapter._config = new_config - self._bootstrap_core.command_tools._config = new_config - self._bootstrap_core.command_tools._allowlisted_commands = set( - new_config.commands.allowlisted_commands - ) - self._bootstrap_core.file_tools._config = new_config - bootstrap_registry = getattr( - self._bootstrap_core._adapter, "_tool_registry", None - ) - if bootstrap_registry is not None: - bootstrap_registry.set_allowed_tools(new_config.tools) - for session_id, session in self._sessions.items(): - session.core._config = new_config - session.core._adapter._config = new_config - session.core.command_tools._config = new_config - session.core.command_tools._allowlisted_commands = set( - new_config.commands.allowlisted_commands - ) - session.core.file_tools._config = new_config - tool_registry = getattr(session.core._adapter, "_tool_registry", None) - if tool_registry is not None: - tool_registry.set_allowed_tools(new_config.tools) + # Rebuild every enforcement point (ToolRegistry allowed_tools / + # approval, command allowlist, adapter provider) rather than + # mutating _config in place, which would leave already-constructed + # adapters enforcing stale policy. This subsumes upstream's + # set_allowed_tools live-update: apply_config refreshes allowed + # tools plus approval/command/provider state. + self._bootstrap_core.apply_config(new_config) + for session in self._sessions.values(): + session.core.apply_config(new_config) logger.info( "Updated %d active session(s) with server agent config.", len(self._sessions), @@ -881,7 +994,6 @@ async def close_session( session_id: str, **kwargs: Any, ) -> None: - await self._require_authenticated() session = self._sessions.get(session_id) if session is None: raise RequestError.invalid_params({"reason": "unknown_session"}) @@ -895,7 +1007,6 @@ async def cancel( session_id: str, **kwargs: Any, ) -> None: - await self._require_authenticated() session = self._sessions.get(session_id) if session is not None and session.active_prompt_task is not None: session.cancel_event.set() @@ -907,18 +1018,41 @@ async def prompt( message_id: str | None = None, **kwargs: Any, ) -> PromptResponse: - await self._require_authenticated() - try: - await asyncio.to_thread(self._authorization.validate) - except AcpAuthorizationFailure: - self._authorized_workspace = None - raise RequestError.auth_required( - {"reason": "authorization_rejected"} - ) from None request_id = message_id or uuid4().hex session = self._sessions.get(session_id) if session is None: raise RequestError.auth_required({"reason": "unknown_session"}) + session_authorization = session.authorization or self._authorization + reauthenticated = False + try: + await asyncio.to_thread(session_authorization.validate) + except AcpAuthorizationFailure: + if config.managed_mode: + try: + await asyncio.to_thread( + session_authorization.prepare_workspace, + session.core._config.workspace_root, + ) + await asyncio.to_thread(session_authorization.authenticate) + reauthenticated = True + except AcpAuthorizationFailure: + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + else: + self._authorized_workspace = None + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + if reauthenticated: + refreshed_config = replace( + session.core._config, + upload=replace( + session.core._config.upload, + auth_headers=session_authorization.telemetry_headers(), + ), + ) + session.core.apply_config(refreshed_config) async with session.prompt_lock: session.cancel_event.clear() session.active_prompt_task = asyncio.current_task() @@ -943,12 +1077,33 @@ async def prompt( ) prompt_text = await _prompt_text_async(prompt) logger.info("Prompt text: %s", prompt_text) + run_id = uuid4().hex + if config.managed_mode: + run_payload = await asyncio.to_thread( + session_authorization.create_managed_run, + run_id=run_id, + session_id=session_id, + ) + from code4me2_agent.config import ServerAgentConfig + + try: + run_policy = ServerAgentConfig.from_managed_payload(run_payload) + except ValueError: + raise RequestError.invalid_params( + {"reason": "managed_run_policy_invalid"} + ) from None + session.core.apply_config( + session.core._config.with_server_overrides( + run_policy, + backend_url=session_authorization.backend_url, + ) + ) result = await asyncio.to_thread( session.core.handle_prompt, prompt_text, request_id, message_id, - None, + run_id, session.cancel_event, ) await asyncio.to_thread(self._save_persisted_memory, session) diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py index 34b4c5d..52fd6f4 100644 --- a/src/code4me2_agent/adapters.py +++ b/src/code4me2_agent/adapters.py @@ -1,19 +1,25 @@ from __future__ import annotations import json +import logging import os +import platform import re from dataclasses import asdict, dataclass, is_dataclass -from time import perf_counter, sleep +from pathlib import Path +from time import perf_counter, sleep, time from typing import TYPE_CHECKING, Any, Protocol + from openai import APIError, APIStatusError, OpenAI -import logging -import fcntl -from pathlib import Path -from threading import Event -from time import time + +try: + import fcntl +except ImportError: # Windows + fcntl = None # type: ignore[assignment] if TYPE_CHECKING: + from threading import Event + from code4me2_agent.command_tools import WorkspaceCommandTools from code4me2_agent.config import AgentConfig from code4me2_agent.events import AgentEventSink @@ -114,7 +120,8 @@ def _rate_limit_provider_request( state_path.parent.mkdir(parents=True, exist_ok=True) lock_path = state_path.with_suffix(state_path.suffix + ".lock") with lock_path.open("a+", encoding="utf-8") as lock_file: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) try: while True: now = float(now_fn()) @@ -137,7 +144,8 @@ def _rate_limit_provider_request( ) sleep_fn(sleep_seconds) finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) def _read_rate_limit_timestamps(state_path: Path) -> list[float]: @@ -230,9 +238,10 @@ def __init__( file_tools: WorkspaceFileTools, command_tools: WorkspaceCommandTools, *, - allowed_tools: list[str] | None = None, + allowed_tools: frozenset[str] | list[str] | None = None, event_sink: AgentEventSink | None = None, mcp_tools: StdioMcpToolBroker | None = None, + approval_policy: str = "auto", ) -> None: from code4me2_agent.events import ( NoopAgentEventSink, @@ -243,7 +252,8 @@ def __init__( self._file_tools = file_tools self._command_tools = command_tools self._mcp_tools = mcp_tools - self._allowed_tools = set(allowed_tools) if allowed_tools is not None else None + self._allowed_tools = frozenset(allowed_tools) if allowed_tools is not None else None + self._approval_policy = approval_policy self._event_sink = event_sink or NoopAgentEventSink() self._event_type = ToolCallEvent self._thought_event_type = ThoughtEvent @@ -252,12 +262,46 @@ def execute( self, tool_call: ToolCall, *, run_id: str, request_id: str ) -> dict[str, Any]: name = tool_call.name - if self._allowed_tools is not None and name not in self._allowed_tools: + mcp_wildcard_allowed = ( + name.startswith("mcp__") + and self._allowed_tools is not None + and "mcp__*" in self._allowed_tools + ) + if ( + self._allowed_tools is not None + and name not in self._allowed_tools + and not mcp_wildcard_allowed + ): raise ToolRegistryError( - f"Tool is not allowed by the assigned profile: {name}", + f"Tool is disabled by the assigned study policy: {name}", failure_reason="tool_not_allowed", ) + if self._approval_policy not in {"auto", "per_step", "suggestion_only"}: + raise ToolRegistryError( + f"Unknown assigned approval policy: {self._approval_policy}", + failure_reason="invalid_approval_policy", + ) + if self._approval_policy == "suggestion_only" and ( + name in {"create_file", "write_file", "replace_text", "run_command"} + or name.startswith("mcp__") + ): + raise ToolRegistryError( + f"Tool execution is disabled by suggestion-only policy: {name}", + failure_reason="approval_policy_denied", + ) arguments = dict(tool_call.arguments) + if self._approval_policy == "per_step": + request_approval = getattr(self._event_sink, "request_approval", None) + approved = bool( + request_approval(tool_call, arguments) + if callable(request_approval) + else False + ) + if not approved: + raise ToolRegistryError( + f"The user denied approval for tool: {name}", + failure_reason="approval_policy_denied", + ) self._emit_tool_start( tool_call, arguments, run_id=run_id, request_id=request_id ) @@ -343,14 +387,21 @@ def definitions(self) -> list[dict[str, Any]]: definitions.extend(self._mcp_tools.definitions()) if self._allowed_tools is not None: definitions = [ - tool - for tool in definitions - if tool.get("function", {}).get("name") in self._allowed_tools + definition + for definition in definitions + if ( + str(definition.get("function", {}).get("name", "")) + in self._allowed_tools + or ( + "mcp__*" in self._allowed_tools + and str(definition.get("function", {}).get("name", "")).startswith("mcp__") + ) + ) ] return definitions - def set_allowed_tools(self, allowed_tools: list[str] | None) -> None: - self._allowed_tools = set(allowed_tools) if allowed_tools is not None else None + def set_allowed_tools(self, allowed_tools: frozenset[str] | list[str] | None) -> None: + self._allowed_tools = frozenset(allowed_tools) if allowed_tools is not None else None def known_tool_names(self) -> set[str]: names: set[str] = set() @@ -470,6 +521,9 @@ def __init__( timeout_seconds: float, auth_headers: dict[str, str] | None = None, tool_definitions: list[dict[str, Any]] | None = None, + temperature: float | None = None, + session_id: str = "", + managed_request: Any | None = None, ) -> None: self._kind = kind.strip() or "code4me_backend" self._base_url = base_url.rstrip("/") @@ -478,14 +532,42 @@ def __init__( self._timeout_seconds = timeout_seconds self._auth_headers = dict(auth_headers or {}) self._tool_definitions = list(tool_definitions or _tool_definitions()) + self._temperature = temperature + self._session_id = session_id + self._managed_request = managed_request - def generate(self, messages: list[dict[str, Any]]) -> ProviderTurn: + def generate(self, messages: list[dict[str, Any]], *, run_id: str = "") -> ProviderTurn: request_payload = { "model": self._model, "messages": [_to_openai_message(message) for message in messages], "tools": self._tool_definitions, "tool_choice": "auto", } + if self._temperature is not None: + request_payload["temperature"] = self._temperature + + if self._kind == "managed_backend": + if not callable(self._managed_request): + raise BackendProviderRequestError("Managed backend transport is unavailable.") + try: + response_payload = self._managed_request( + run_id=run_id, + session_id=self._session_id, + model_request=request_payload, + ) + except Exception as exc: + raise BackendProviderRequestError( + f"The managed backend model request failed: {exc}" + ) from exc + normalized_output = _normalize_openai_provider_response(response_payload) + return ProviderTurn( + output=normalized_output, + usage=_normalize_usage(response_payload.get("usage"), messages, normalized_output), + finish_reason=_response_finish_reason(response_payload), + model=str(response_payload.get("model") or self._model), + request_payload=request_payload, + raw_response=response_payload, + ) client = self._client() max_429_retries = _non_negative_int_env("CODE4ME_BACKEND_429_MAX_RETRIES", 3) @@ -550,7 +632,7 @@ def _headers(self) -> dict[str, str]: elif self._kind == "openai": if api_key: headers["Authorization"] = f"Bearer {api_key}" - else: + elif self._kind != "managed_backend": raise ValueError(f"Unsupported provider kind: {self._kind}") return headers @@ -691,7 +773,7 @@ def handle_prompt( started_at = perf_counter() # 4 if isinstance(provider, OpenAICompatibleProvider): - provider_turn = provider.generate(messages) + provider_turn = provider.generate(messages, run_id=run_id) self._record_model_completed( run_id=run_id, request_id=request_id, @@ -1031,13 +1113,23 @@ def _provider(self) -> FakeOpenAICompatibleProvider | OpenAICompatibleProvider: timeout_seconds=self._config.adapter.provider.timeout_seconds, auth_headers=self._config.adapter.provider.auth_headers, tool_definitions=self._tool_registry.definitions(), + temperature=self._config.adapter.provider.temperature, + session_id=self._config.session_id, + managed_request=self._config.managed_request, ) def _system_context(self) -> str: + from code4me2_agent.command_tools import available_commands + workspace_root = self._config.workspace_root.as_posix() + executable_commands = available_commands( + self._config.commands.allowlisted_commands + ) + command_summary = ", ".join(executable_commands) or "none" return ( "You are a helpful programming assistant. " f"The current working directory and workspace root is {workspace_root}. " + f"The participant host operating system is {platform.system()}; executable commands allowed by policy are: {command_summary}. " "Respond conversationally to greetings and general questions without calling tools. " "Only use tools when the user explicitly asks you to read, search, create, edit, or list files, " "or to run a command in the workspace. " @@ -1047,6 +1139,7 @@ def _system_context(self) -> str: "Do not output code as text when the user expects you to apply the change. " "When the user asks you to run, execute, compile, or build a file or command, " "you MUST call run_command with the appropriate argv rather than just describing the command. " + "Never assume Bash or Unix utilities are installed when they are absent from the executable command list. " "When calling tools, prefer workspace-relative paths and cwd='.' unless the user explicitly asks for an absolute path. " "Do not invent container paths such as /workspace. " "After file tools complete, answer directly in assistant text; do not run commands just to print a confirmation." @@ -1286,7 +1379,14 @@ def create_agent_adapter( tool_registry=ToolRegistry( file_tools, command_tools, - allowed_tools=config.tools, + allowed_tools=config.allowed_tools + if config.allowed_tools is not None + else ( + frozenset(config.tools) + if getattr(config, "tools", None) is not None + else None + ), + approval_policy=config.approval_policy, event_sink=event_sink, mcp_tools=mcp_tools, ), diff --git a/src/code4me2_agent/cli.py b/src/code4me2_agent/cli.py index eebdc70..93be265 100644 --- a/src/code4me2_agent/cli.py +++ b/src/code4me2_agent/cli.py @@ -5,16 +5,24 @@ import json import logging import os +import platform import shutil import sys from pathlib import Path +from code4me2_agent._build_version import RUNTIME_VERSION from code4me2_agent.acp_runtime import run_acp_stdio -from code4me2_agent.config import AgentConfig - +from code4me2_agent.config import ( + AdapterConfig, + AgentConfig, + CommandConfig, + MemoryWindowConfig, + OpenAICompatibleProviderConfig, +) DEFAULT_CONFIG_PATH = os.path.expanduser("~/.code4me/agent-config.json") DEFAULT_ACP_JSON_PATH = os.path.expanduser("~/.code4me/acp.json") +SUPPORTED_MANAGED_PLATFORMS = {"darwin", "linux", "win32"} # Scaffold written by `code4me2-agent init`. Deliberately specifies no model, # provider or API key: those come from the agent profile the backend assigns @@ -83,7 +91,7 @@ def _resolve_agent_command() -> str: if resolved_command: return resolved_command - for command in ("code4me2-agent"): + for command in ("code4me2-agent",): resolved_command = shutil.which(command) if resolved_command: return resolved_command @@ -109,6 +117,21 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Install default config files to ~/.code4me/ and print setup instructions.", ) + parser.add_argument( + "--managed", + action="store_true", + help="Run using the authenticated Code4Me IDE bridge and hosted backend.", + ) + parser.add_argument( + "--version", + action="store_true", + help="Print the packaged runtime version and exit.", + ) + parser.add_argument( + "--self-check", + action="store_true", + help="Check runtime dependencies and platform support without changing files.", + ) return parser @@ -155,13 +178,83 @@ def main(argv: list[str] | None = None) -> None: configure_logging() args = build_parser().parse_args(argv) + if args.version: + print(_runtime_version()) + return + + if args.self_check: + raise SystemExit(_self_check()) + if args.setup: _do_setup() sys.exit(0) - config = AgentConfig.from_file(args.config) + config = _managed_config() if args.managed else AgentConfig.from_file(args.config) asyncio.run(run_acp_stdio(config)) +def _runtime_version() -> str: + return RUNTIME_VERSION + + +def _managed_config() -> AgentConfig: + workspace = Path.cwd().resolve() + if os.name == "nt": + commands = ["cmd", "powershell", "pwsh", "git", "python", "python3"] + else: + commands = list(DEFAULT_AGENT_CONFIG["commands"]["allowlist"]) + return AgentConfig( + workspace_root=workspace, + trace_path=workspace / ".code4me" / "acp-trace.jsonl", + session_id="managed-bootstrap", + raw_capture_enabled=False, + store_agent_content=False, + commands=CommandConfig(allowlisted_commands=commands), + adapter=AdapterConfig( + name="openai_compatible_react", + max_iterations=1, + memory_window=MemoryWindowConfig( + scope="session", strategy="token_window", max_messages=20, max_tokens=6000 + ), + provider=OpenAICompatibleProviderConfig(kind="managed_backend"), + ), + allowed_tools=frozenset(), + managed_mode=True, + ) + + +def _self_check() -> int: + from code4me2_agent.command_tools import available_commands + + if os.name == "nt": + policy_commands = ["cmd", "powershell", "pwsh", "git", "python", "python3"] + else: + policy_commands = list(DEFAULT_AGENT_CONFIG["commands"]["allowlist"]) + checks = { + "runtime_version": _runtime_version(), + "platform": sys.platform, + "platform_supported": sys.platform in SUPPORTED_MANAGED_PLATFORMS, + "python": platform.python_version(), + "frozen": bool(getattr(sys, "frozen", False)), + "os": platform.system(), + "architecture": platform.machine(), + "policy_commands": policy_commands, + "available_commands": available_commands(policy_commands), + } + try: + import acp # noqa: F401 + import openai # noqa: F401 + from acp.agent.router import build_agent_router # noqa: F401 + from mcp import Client, StdioServerParameters # noqa: F401 + + checks["dependencies"] = "ok" + except ImportError as exc: + checks["dependencies"] = f"missing: {exc.name}" + ok = checks["platform_supported"] and checks["dependencies"] == "ok" + checks["status"] = "ok" if ok else "failed" + print(json.dumps(checks, sort_keys=True)) + return 0 if ok else 1 + + if __name__ == "__main__": main() diff --git a/src/code4me2_agent/command_tools.py b/src/code4me2_agent/command_tools.py index ea1a73a..4b47d0d 100644 --- a/src/code4me2_agent/command_tools.py +++ b/src/code4me2_agent/command_tools.py @@ -2,6 +2,8 @@ import asyncio import inspect +import os +import shutil import subprocess from dataclasses import dataclass from pathlib import Path @@ -393,6 +395,7 @@ def _run_local( errors="replace", timeout=self._timeout_seconds, check=False, + env=_external_command_environment(), ) except subprocess.TimeoutExpired as exc: return ( @@ -409,6 +412,7 @@ def _run_local( False, ) + def _acp_run_command(self) -> Any | None: if not bool(getattr(self._acp_backend, "terminal_enabled", False)): return None @@ -638,6 +642,25 @@ def _record_tool_event( payload=payload, raw_payload=None, ) + + +def available_commands(commands: list[str]) -> list[str]: + """Return policy commands which can actually launch on this machine.""" + return [command for command in commands if shutil.which(command)] + + +def _external_command_environment() -> dict[str, str]: + """Undo PyInstaller loader changes before spawning participant tools.""" + environment = dict(os.environ) + environment.pop("_MEIPASS2", None) + original_library_path = environment.pop("LD_LIBRARY_PATH_ORIG", None) + if original_library_path is None: + environment.pop("LD_LIBRARY_PATH", None) + else: + environment["LD_LIBRARY_PATH"] = original_library_path + return environment + + def _resolve_method(client: object, names: list[str]) -> Any | None: for name in names: method = getattr(client, name, None) diff --git a/src/code4me2_agent/config.py b/src/code4me2_agent/config.py index 01bf93a..880aa6e 100644 --- a/src/code4me2_agent/config.py +++ b/src/code4me2_agent/config.py @@ -3,6 +3,7 @@ import json from dataclasses import dataclass, field from pathlib import Path +from typing import Callable from uuid import uuid4 @@ -13,6 +14,9 @@ class UploadConfig: batch_size: int = 50 timeout_seconds: float = 5.0 auth_headers: dict[str, str] = field(default_factory=dict) + auth_headers_provider: Callable[[], dict[str, str]] | None = field( + default=None, repr=False, compare=False + ) @dataclass(frozen=True) @@ -69,6 +73,7 @@ class OpenAICompatibleProviderConfig: model: str = "" api_key_env: str = "" timeout_seconds: float = 90.0 + temperature: float | None = None auth_headers: dict[str, str] = field(default_factory=dict) @property @@ -114,6 +119,52 @@ class ServerAgentConfig: # capture, which the agent has no power to do. store_agent_content: bool = True + @classmethod + def from_managed_payload(cls, payload: dict) -> "ServerAgentConfig": + """Parse a complete protocol-v1 policy, failing closed on bad fields.""" + nested_policy = payload.get("policy_snapshot") or payload.get("policy") + merged = {**payload, **nested_policy} if isinstance(nested_policy, dict) else payload + protocol = merged.get("managed_protocol_version", merged.get("version")) + tools = merged.get("tools") + commands = merged.get("commands_allowlist") + iterations = merged.get("max_iterations") + context_tokens = merged.get("max_context_tokens") + temperature = merged.get("temperature") + required_text = ("agent_profile", "framework_version", "model", "transport") + if protocol != "1" or any( + not isinstance(merged.get(key), str) or not merged[key].strip() + for key in required_text + ): + raise ValueError("Managed agent policy metadata is incomplete.") + if merged["framework_version"] not in {"code4me2-agent", "code4me-agent"}: + raise ValueError("Managed agent policy selects an unsupported runtime.") + if merged["transport"] != "managed_backend": + raise ValueError("Managed agent policy selects an unsafe transport.") + if ( + not isinstance(tools, list) + or not all(isinstance(tool, str) and tool.strip() for tool in tools) + or not isinstance(commands, list) + or not all(isinstance(command, str) and command.strip() for command in commands) + or isinstance(iterations, bool) + or not isinstance(iterations, int) + or iterations < 1 + or isinstance(context_tokens, bool) + or not isinstance(context_tokens, int) + or context_tokens < 1 + or merged.get("approval_policy") not in {"auto", "per_step", "suggestion_only"} + or isinstance(temperature, bool) + or ( + temperature is not None + and ( + not isinstance(temperature, (int, float)) + or not 0.0 <= float(temperature) <= 2.0 + ) + ) + or not isinstance(merged.get("store_agent_content"), bool) + ): + raise ValueError("Managed agent policy contains invalid executable settings.") + return cls.from_payload(merged) + @classmethod def from_payload(cls, payload: dict) -> "ServerAgentConfig": """Parse an agent-config response, ignoring anything malformed. @@ -123,6 +174,10 @@ def from_payload(cls, payload: dict) -> "ServerAgentConfig": config still beats falling back to a local guess. """ + nested_policy = payload.get("policy_snapshot") or payload.get("policy") + if isinstance(nested_policy, dict): + payload = {**payload, **nested_policy} + def _clean_str(key: str) -> str | None: value = payload.get(key) if isinstance(value, str) and value.strip(): @@ -174,6 +229,8 @@ def has_overrides(self) -> bool: self.tools, self.max_iterations, self.max_context_tokens, + self.approval_policy, + self.temperature, ) ) @@ -188,6 +245,13 @@ class AgentConfig: upload: UploadConfig = field(default_factory=UploadConfig) commands: CommandConfig = field(default_factory=CommandConfig) adapter: AdapterConfig = field(default_factory=AdapterConfig) + allowed_tools: frozenset[str] | None = None + approval_policy: str = "auto" + store_agent_content: bool = True + managed_mode: bool = False + managed_request: Callable[[str, str, dict | None], dict] | None = field( + default=None, repr=False, compare=False + ) def with_server_overrides( self, server: ServerAgentConfig, *, backend_url: str | None = None @@ -206,17 +270,31 @@ def with_server_overrides( provider = self.adapter.provider - if backend_url: + if self.managed_mode and backend_url: + provider = _replace( + provider, + kind="managed_backend", + base_url=str(backend_url).rstrip("/"), + api_key_env="", + auth_headers={}, + ) + elif backend_url: provider = _replace( provider, kind="code4me_backend", base_url=str(backend_url).rstrip("/"), ) + elif server.base_url: + provider = _replace( + provider, kind="openai", base_url=server.base_url + ) if server.model: provider = _replace(provider, model=server.model) - if server.api_key_ref and provider.kind != "code4me_backend": + if server.api_key_ref and provider.kind not in ("code4me_backend", "managed_backend"): provider = _replace(provider, api_key_env=server.api_key_ref) + if server.temperature is not None: + provider = _replace(provider, temperature=server.temperature) memory_window = self.adapter.memory_window if server.max_context_tokens is not None: @@ -233,15 +311,28 @@ def with_server_overrides( commands = self.commands if server.commands_allowlist is not None: + from code4me2_agent.command_tools import available_commands + commands = _replace( - commands, allowlisted_commands=list(server.commands_allowlist) + commands, + allowlisted_commands=available_commands(server.commands_allowlist), ) tools = self.tools + allowed_tools = self.allowed_tools if server.tools is not None: tools = list(server.tools) + allowed_tools = frozenset(server.tools) - return _replace(self, adapter=adapter, commands=commands, tools=tools) + return _replace( + self, + adapter=adapter, + commands=commands, + tools=tools, + allowed_tools=allowed_tools, + approval_policy=server.approval_policy or self.approval_policy, + store_agent_content=server.store_agent_content, + ) @classmethod def from_file(cls, config_path: str | Path) -> "AgentConfig": @@ -413,6 +504,12 @@ def from_file(cls, config_path: str | Path) -> "AgentConfig": provider_data.get("timeout_seconds", 90.0), default=90.0, ), + temperature=( + float(provider_data["temperature"]) + if isinstance(provider_data.get("temperature"), (int, float)) + and not isinstance(provider_data.get("temperature"), bool) + else None + ), auth_headers=provider_auth_headers, ), ) diff --git a/src/code4me2_agent/echo.py b/src/code4me2_agent/echo.py index 7998201..d667ef1 100644 --- a/src/code4me2_agent/echo.py +++ b/src/code4me2_agent/echo.py @@ -3,7 +3,6 @@ import logging from dataclasses import dataclass from time import perf_counter -from threading import Event from typing import TYPE_CHECKING from uuid import uuid4 @@ -15,6 +14,8 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: + from threading import Event + from code4me2_agent.config import AgentConfig from code4me2_agent.events import AgentEventSink from code4me2_agent.mcp_tools import StdioMcpToolBroker @@ -40,6 +41,9 @@ def __init__( self.file_tools = WorkspaceFileTools(config, telemetry=self._telemetry) self.command_tools = WorkspaceCommandTools(config, telemetry=self._telemetry) self._session_memory = self._build_session_memory() + self._acp_file_backend = None + self._acp_command_backend = None + self._mcp_tools = None self._adapter = create_agent_adapter( config, telemetry=self._telemetry, @@ -55,6 +59,9 @@ def rebuild_tools( acp_command_backend: object | None = None, mcp_tools: StdioMcpToolBroker | None = None, ) -> None: + self._acp_file_backend = acp_file_backend + self._acp_command_backend = acp_command_backend + self._mcp_tools = mcp_tools self.file_tools = WorkspaceFileTools( self._config, acp_backend=acp_file_backend, @@ -67,6 +74,23 @@ def rebuild_tools( ) self.rebuild_adapter(mcp_tools=mcp_tools) + def apply_config(self, config: AgentConfig) -> None: + """Apply an immutable server policy and rebuild every enforcement point.""" + previous_memory = self.session_memory_snapshot() + self._config = config + self._telemetry.apply_config(config) + # Memory limits are part of the assigned A/B policy. Recreate the + # window so a policy refresh cannot leave an existing session using + # the previous arm's context budget, while retaining its conversation. + self._session_memory = self._build_session_memory() + if self._session_memory is not None and previous_memory is not None: + self._session_memory.replace_messages(previous_memory) + self.rebuild_tools( + acp_file_backend=self._acp_file_backend, + acp_command_backend=self._acp_command_backend, + mcp_tools=self._mcp_tools, + ) + def rebuild_adapter( self, *, diff --git a/src/code4me2_agent/mcp_tools.py b/src/code4me2_agent/mcp_tools.py index d86ff10..e506283 100644 --- a/src/code4me2_agent/mcp_tools.py +++ b/src/code4me2_agent/mcp_tools.py @@ -7,14 +7,16 @@ import re from contextlib import AsyncExitStack from dataclasses import dataclass -from pathlib import Path from threading import Event, Thread -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable from mcp import Client, StdioServerParameters from code4me2_agent.acp_utils import capability_value +if TYPE_CHECKING: + from pathlib import Path + logger = logging.getLogger(__name__) _MCP_STARTUP_TIMEOUT_SECONDS = 30.0 diff --git a/src/code4me2_agent/runtime_auth.py b/src/code4me2_agent/runtime_auth.py index 76b030e..9526957 100644 --- a/src/code4me2_agent/runtime_auth.py +++ b/src/code4me2_agent/runtime_auth.py @@ -3,10 +3,17 @@ import json import logging import os +import platform +import stat +import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from urllib import error, request +from urllib.parse import urlparse + +if TYPE_CHECKING: + from code4me2_agent.config import ServerAgentConfig logger = logging.getLogger(__name__) @@ -27,6 +34,10 @@ class AcpAuthorizationFailure(Exception): pass +class AcpSessionExpired(AcpAuthorizationFailure): + """The backend specifically rejected a bearer token as invalid/expired.""" + + @dataclass(frozen=True) class AcpRuntimeScope: project_id: str @@ -34,6 +45,7 @@ class AcpRuntimeScope: class AcpBackendAuthorization: + needs_workspace = False def __init__( self, *, @@ -331,14 +343,15 @@ def authorized_json_request( with request.urlopen(http_request, timeout=timeout) as response: return json.loads(response.read().decode("utf-8")) except error.HTTPError as exc: - if exc.code == 404: + if exc.code == 404 and method.upper() == "GET": return {} logger.warning( "ACP authorized request failed with HTTP status %s url=%s.", exc.code, http_request.full_url, ) - raise AcpAuthorizationFailure( + failure_type = AcpSessionExpired if exc.code == 401 else AcpAuthorizationFailure + raise failure_type( "Code4Me ACP authorized request was rejected." ) from None except error.URLError as exc: @@ -361,6 +374,21 @@ def authorized_json_request( "Code4Me ACP authorized request was rejected." ) from None + def create_managed_run(self, *, run_id: str, session_id: str) -> dict: + return self.authorized_json_request( + "POST", "/api/acp/runs", {"run_id": run_id, "session_id": session_id} + ) + + def managed_inference( + self, *, run_id: str, session_id: str, model_request: dict + ) -> dict: + return self.authorized_json_request( + "POST", + "/api/acp/inference", + {"run_id": run_id, "session_id": session_id, "request": model_request}, + timeout=120.0, + ) + def _refresh_prepared_credentials(self) -> None: if self._workspace_root is None: return @@ -589,3 +617,205 @@ def _read_runtime_handoff_file( ) -> tuple[str | None, str | None]: values = _read_runtime_auth_values(workspace_root, include_home=include_home) return values.get("CODE4ME_ACP_BACKEND_URL"), values.get("CODE4ME_ACP_GRANT") + + +def _path_format() -> str: + return "windows" if os.name == "nt" else "posix" + + +def _bridge_directories() -> list[Path]: + override = os.environ.get("CODE4ME_BRIDGE_DIR", "").strip() + if override: + return [Path(override).expanduser()] + if os.name == "nt": + root = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return [root / "Code4Me" / "bridges"] + if platform.system() == "Darwin": + return [Path.home() / "Library" / "Caches" / "JetBrains" / "Code4Me" / "bridges"] + candidates: list[Path] = [] + runtime_dir = os.environ.get("XDG_RUNTIME_DIR", "").strip() + if runtime_dir: + candidates.append(Path(runtime_dir) / "code4me" / "bridges") + candidates.append(Path.home() / ".cache" / "JetBrains" / "Code4Me" / "bridges") + return candidates + + +def _record_is_private(path: Path) -> bool: + if os.name == "nt": + return True + try: + return not bool(stat.S_IMODE(path.stat().st_mode) & 0o077) + except OSError: + return False + + +def _canonical_workspace(value: str | Path) -> str: + return os.path.normcase(str(Path(value).expanduser().resolve())) + + +@dataclass(frozen=True) +class PluginBridge: + endpoint: str + capability: str + record_path: Path + + +def _bridge_is_live(bridge: PluginBridge) -> bool: + status_request = request.Request( + f"{bridge.endpoint}/v1/status", + headers={"Authorization": f"Bearer {bridge.capability}"}, + method="GET", + ) + try: + with request.urlopen(status_request, timeout=0.75) as response: + payload = json.loads(response.read().decode("utf-8")) + return ( + response.status == 200 + and isinstance(payload, dict) + and payload.get("protocol_version") == "1" + ) + except (error.HTTPError, error.URLError, TimeoutError, ValueError, OSError): + return False + + +def discover_plugin_bridge(workspace: str | Path) -> PluginBridge: + """Return the single private, loopback-only IDE bridge for a workspace.""" + expected = _canonical_workspace(workspace) + matches: list[PluginBridge] = [] + for directory in _bridge_directories(): + if not directory.is_dir(): + continue + for record_path in directory.glob("*.json"): + if not _record_is_private(record_path): + logger.warning("Ignoring unsafe bridge record %s", record_path) + continue + try: + payload = json.loads(record_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(payload, dict) or str(payload.get("protocol_version")) != "1": + continue + raw_workspaces = payload.get("workspaces", []) + claimed: list[str] = [] + if isinstance(raw_workspaces, list): + for item in raw_workspaces: + value = item.get("workspace") if isinstance(item, dict) else item + if isinstance(value, str): + claimed.append(_canonical_workspace(value)) + if expected not in claimed: + continue + endpoint = str(payload.get("base_url") or "").rstrip("/") + capability = str(payload.get("capability") or "") + parsed = urlparse(endpoint) + if ( + parsed.scheme != "http" + or parsed.hostname not in {"127.0.0.1", "::1", "localhost"} + or not capability + ): + continue + bridge = PluginBridge(endpoint, capability, record_path) + if _bridge_is_live(bridge): + matches.append(bridge) + if not matches: + raise AcpAuthorizationFailure( + "No authenticated Code4Me IDE instance is available for this workspace." + ) + if len(matches) != 1: + raise AcpAuthorizationFailure( + "Multiple Code4Me IDE instances claim this workspace." + ) + return matches[0] + + +class ManagedBridgeAuthorization(AcpBackendAuthorization): + """Acquire short-lived backend credentials from the local plugin bridge.""" + + needs_workspace = True + + def __init__(self) -> None: + super().__init__(backend_url=None, grant=None, workspace_root=None) + self._launch_id = uuid.uuid4().hex + + def prepare_workspace(self, workspace: str | Path) -> None: + resolved = Path(workspace).expanduser().resolve() + if self._workspace_root == resolved and self.is_authenticated: + return + bridge = discover_plugin_bridge(resolved) + payload = { + "workspace": str(resolved), + "path_format": _path_format(), + "managed_protocol_version": "1", + "launch_id": self._launch_id, + } + http_request = request.Request( + f"{bridge.endpoint}/v1/grant", + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {bridge.capability}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with request.urlopen(http_request, timeout=5.0) as response: + bridge_payload = json.loads(response.read().decode("utf-8")) + except (error.HTTPError, error.URLError, TimeoutError, ValueError) as exc: + raise AcpAuthorizationFailure( + "The Code4Me IDE authentication bridge rejected this workspace." + ) from exc + backend_url = bridge_payload.get("backend_url") if isinstance(bridge_payload, dict) else None + grant = bridge_payload.get("grant") if isinstance(bridge_payload, dict) else None + if not backend_url or not grant: + raise AcpAuthorizationFailure("The Code4Me IDE returned an invalid runtime grant.") + self._workspace_root = resolved + self._backend_url = str(backend_url).rstrip("/") + self._grant = str(grant) + self._acp_token = None + self._scope = None + + def _refresh_prepared_credentials(self) -> None: + return + + def _write_session_cache(self) -> None: + return + + def authorized_json_request( + self, + method: str, + path: str, + payload: dict | None = None, + *, + timeout: float = 10.0, + ) -> dict: + try: + return super().authorized_json_request( + method, path, payload, timeout=timeout + ) + except AcpSessionExpired: + if self._workspace_root is None: + raise + workspace = self._workspace_root + self._acp_token = None + self._scope = None + self.prepare_workspace(workspace) + self.authenticate() + return super().authorized_json_request( + method, path, payload, timeout=timeout + ) + + def fetch_agent_config(self) -> None: + if not self._backend_url or not self._acp_token: + raise AcpAuthorizationFailure("Managed agent configuration requires authentication.") + self._server_agent_config = None + payload = self._get( + "/api/acp/agent-config?managed_protocol_version=1", + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + from code4me2_agent.config import ServerAgentConfig + + try: + self._server_agent_config = ServerAgentConfig.from_managed_payload(payload) + except ValueError as exc: + raise AcpAuthorizationFailure( + "The Code4Me server returned an invalid managed-agent policy." + ) from exc diff --git a/src/code4me2_agent/telemetry.py b/src/code4me2_agent/telemetry.py index 6ca278b..20527eb 100644 --- a/src/code4me2_agent/telemetry.py +++ b/src/code4me2_agent/telemetry.py @@ -2,7 +2,7 @@ import json from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Callable, Protocol from urllib import request from uuid import uuid4 @@ -14,6 +14,33 @@ SCHEMA_VERSION = "code4me.agent.event.v1" +_CONTENT_PAYLOAD_KEYS = frozenset( + { + "arguments", + "argv", + "content", + "error_message", + "messages", + "new_text", + "old_text", + "query", + "request", + "response", + "result", + "stderr", + "stdout", + "text", + } +) + + +def _payload_without_content(payload: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in payload.items() + if key not in _CONTENT_PAYLOAD_KEYS + } + # TODO this file should be working with the database. and may be moved to another folder. class TelemetrySink(Protocol): def append(self, event: dict[str, Any]) -> None: @@ -50,12 +77,17 @@ def __init__( upload_events: UploadEventsCallable, batch_size: int = 50, auth_headers: dict[str, str] | None = None, + auth_headers_provider: Callable[[], dict[str, str]] | None = None, ) -> None: self._upload_events = upload_events self._batch_size = max(1, int(batch_size)) self._auth_headers = dict(auth_headers or {}) + self._auth_headers_provider = auth_headers_provider self._pending_run_events: dict[str, list[dict[str, Any]]] = {} + def set_auth_headers(self, auth_headers: dict[str, str]) -> None: + self._auth_headers = dict(auth_headers) + def append(self, event: dict[str, Any]) -> None: run_id = event["run_id"] pending_events = self._pending_run_events.setdefault(run_id, []) @@ -70,7 +102,12 @@ def _flush_ready_runs(self) -> None: run_payload = _build_run_payload(pending_events) try: - self._upload_events(run_payload, pending_events, dict(self._auth_headers)) + headers = ( + self._auth_headers_provider() + if callable(self._auth_headers_provider) + else dict(self._auth_headers) + ) + self._upload_events(run_payload, pending_events, dict(headers)) except Exception: continue del self._pending_run_events[run_id] @@ -149,13 +186,18 @@ def upload_events( upload_events=upload_events, batch_size=config.upload.batch_size, auth_headers=config.upload.auth_headers, + auth_headers_provider=config.upload.auth_headers_provider, ) class AgentTelemetryRecorder: def __init__(self, config: AgentConfig, sinks: list[TelemetrySink] | None = None) -> None: self._config = config - self._jsonl = JsonlTelemetrySink(config.trace_path) + # Participant mode must not leave study prompts, tool arguments, or a + # surprise .code4me directory in the participant's project. The + # authenticated server sink remains the telemetry system of record; + # local JSONL is retained only for explicit developer configurations. + self._jsonl = None if config.managed_mode else JsonlTelemetrySink(config.trace_path) if sinks is None: upload_sink = build_upload_sink(config) self._sinks = [upload_sink] if upload_sink is not None else [] @@ -163,6 +205,14 @@ def __init__(self, config: AgentConfig, sinks: list[TelemetrySink] | None = None self._sinks = sinks self._run_sequences: dict[str, int] = {} + def apply_config(self, config: AgentConfig) -> None: + """Refresh policy metadata and rotating bearer headers without dropping events.""" + self._config = config + for sink in self._sinks: + set_auth_headers = getattr(sink, "set_auth_headers", None) + if callable(set_auth_headers): + set_auth_headers(config.upload.auth_headers) + def record( self, *, @@ -181,6 +231,11 @@ def record( self._run_sequences[run_id] = sequence observed_tool = str(payload.get("tool_name", "none")) + stored_payload = ( + payload + if self._config.store_agent_content + else _payload_without_content(payload) + ) event = { "schema_version": SCHEMA_VERSION, "event_id": uuid4().hex, @@ -192,8 +247,13 @@ def record( "run_id": run_id, "request_id": request_id, "parent_event_id": parent_event_id, - "payload": payload, - "raw_payload": raw_payload if self._config.raw_capture_enabled else None, + "payload": stored_payload, + "raw_payload": ( + raw_payload + if self._config.raw_capture_enabled + and self._config.store_agent_content + else None + ), "metrics": metrics or {}, "privacy": { "capture_mode": "raw" if self._config.raw_capture_enabled else "redacted", @@ -213,7 +273,8 @@ def record( if message_id is not None: event["message_id"] = message_id - self._jsonl.append(event) + if self._jsonl is not None: + self._jsonl.append(event) for sink in self._sinks: sink.append(event) return event diff --git a/tests/code4me2_agent/__init__.py b/tests/code4me2_agent/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/tests/code4me2_agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/code4me2_agent/test_cli.py b/tests/code4me2_agent/test_cli.py new file mode 100644 index 0000000..9654d67 --- /dev/null +++ b/tests/code4me2_agent/test_cli.py @@ -0,0 +1,18 @@ +from unittest.mock import patch + +from code4me2_agent.cli import _resolve_agent_command + + +def test_resolve_agent_command_checks_the_full_console_script_name(): + with patch("code4me2_agent.cli.sys.argv", ["unavailable-launcher"]), patch( + "code4me2_agent.cli.shutil.which", + side_effect=lambda command: "/tools/code4me2-agent" + if command == "code4me2-agent" + else None, + ) as which: + assert _resolve_agent_command() == "/tools/code4me2-agent" + + assert [call.args[0] for call in which.call_args_list] == [ + "unavailable-launcher", + "code4me2-agent", + ] diff --git a/tests/code4me2_agent/test_policy_enforcement.py b/tests/code4me2_agent/test_policy_enforcement.py new file mode 100644 index 0000000..47bf66b --- /dev/null +++ b/tests/code4me2_agent/test_policy_enforcement.py @@ -0,0 +1,292 @@ +from dataclasses import replace +from unittest.mock import MagicMock, patch + +import pytest + +from code4me2_agent.adapters import ( + OpenAICompatibleReactAdapter, + ToolCall, + ToolRegistry, + ToolRegistryError, +) +from code4me2_agent.config import AgentConfig, MemoryWindowConfig, ServerAgentConfig +from code4me2_agent.echo import EchoAgentCore +from code4me2_agent.runtime_auth import ( + AcpBackendAuthorization, + AcpRuntimeScope, + AcpSessionExpired, + ManagedBridgeAuthorization, +) +from code4me2_agent.telemetry import AgentTelemetryRecorder, ServerUploadTelemetrySink + + +class ApprovalSink: + def __init__(self, approved: bool): + self.approved = approved + self.requests = [] + + def request_approval(self, tool_call, arguments): + self.requests.append((tool_call, arguments)) + return self.approved + + def tool_call(self, event): + return None + + +def test_per_step_policy_denies_before_tool_execution(): + file_tools = MagicMock() + sink = ApprovalSink(False) + registry = ToolRegistry( + file_tools, + MagicMock(), + event_sink=sink, + allowed_tools=frozenset({"read_file"}), + approval_policy="per_step", + ) + + with pytest.raises(ToolRegistryError) as error: + registry.execute( + ToolCall("call-1", "read_file", {"path": "README.md"}), + run_id="run-1", + request_id="request-1", + ) + + assert error.value.failure_reason == "approval_policy_denied" + file_tools.read_file.assert_not_called() + assert len(sink.requests) == 1 + + +def test_per_step_policy_executes_after_allow_once(): + file_tools = MagicMock() + file_tools.read_file.return_value = {"content": "ok"} + registry = ToolRegistry( + file_tools, + MagicMock(), + event_sink=ApprovalSink(True), + allowed_tools=frozenset({"read_file"}), + approval_policy="per_step", + ) + + result = registry.execute( + ToolCall("call-1", "read_file", {"path": "README.md"}), + run_id="run-1", + request_id="request-1", + ) + + assert result == {"content": "ok"} + file_tools.read_file.assert_called_once() + + +def test_suggestion_only_denies_unknown_mcp_side_effects(): + mcp = MagicMock() + mcp.has_tool.return_value = True + registry = ToolRegistry( + MagicMock(), + MagicMock(), + mcp_tools=mcp, + allowed_tools=frozenset({"mcp__issue_tracker__create"}), + approval_policy="suggestion_only", + ) + + with pytest.raises(ToolRegistryError) as error: + registry.execute( + ToolCall("call-1", "mcp__issue_tracker__create", {}), + run_id="run-1", + request_id="request-1", + ) + + assert error.value.failure_reason == "approval_policy_denied" + mcp.execute.assert_not_called() + + +def test_managed_request_reacquires_once_after_expired_bearer(tmp_path): + authorization = ManagedBridgeAuthorization() + authorization._workspace_root = tmp_path + authorization._backend_url = "https://example.test" + authorization._acp_token = "expired" + authorization._scope = AcpRuntimeScope("project", str(tmp_path)) + + with patch.object( + AcpBackendAuthorization, + "authorized_json_request", + side_effect=[AcpSessionExpired("expired"), {"ok": True}], + ), patch.object(authorization, "prepare_workspace") as prepare, patch.object( + authorization, "authenticate" + ) as authenticate: + result = authorization.authorized_json_request("POST", "/api/acp/runs", {}) + + assert result == {"ok": True} + prepare.assert_called_once_with(tmp_path) + authenticate.assert_called_once_with() + + +def test_telemetry_resolves_rotating_bearer_when_flushing(): + uploaded = [] + token = {"value": "first"} + sink = ServerUploadTelemetrySink( + lambda run, events, headers: uploaded.append(headers), + auth_headers_provider=lambda: {"Authorization": f"Bearer {token['value']}"}, + ) + token["value"] = "refreshed" + + sink.append( + { + "run_id": "run-1", + "session_id": "session-1", + "source": "code4me2_agent", + "timestamp": "2026-09-10T00:00:00Z", + "event_type": "agent.run.completed", + "payload": {"status": "completed"}, + } + ) + + assert uploaded == [{"Authorization": "Bearer refreshed"}] + + +def test_policy_refresh_rebuilds_session_memory_limit(tmp_path): + config = AgentConfig( + workspace_root=tmp_path, + trace_path=tmp_path / "trace.jsonl", + session_id="session-1", + ) + config = replace( + config, + adapter=replace( + config.adapter, + memory_window=MemoryWindowConfig( + scope="session", + strategy="token_window", + max_messages=20, + max_tokens=100, + ), + ), + ) + core = EchoAgentCore(config) + core.load_session_memory( + [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "first message"}, + {"role": "assistant", "content": "second message"}, + ] + ) + restricted = replace( + config, + adapter=replace( + config.adapter, + memory_window=replace(config.adapter.memory_window, max_tokens=2), + ), + ) + + core.apply_config(restricted) + + assert core._session_memory is not None + assert core._session_memory.window() == [ + {"role": "system", "content": "system"}, + {"role": "assistant", "content": "second message"}, + ] + + +def test_managed_policy_parser_rejects_partial_or_direct_provider_config(): + with pytest.raises(ValueError, match="metadata"): + ServerAgentConfig.from_managed_payload({"model": "model"}) + + policy = { + "version": "1", + "transport": "openai", + "agent_profile": "arm-a", + "framework_version": "code4me2-agent", + "model": "model", + "tools": [], + "commands_allowlist": [], + "max_iterations": 3, + "max_context_tokens": 1000, + "approval_policy": "auto", + "temperature": None, + "store_agent_content": False, + } + with pytest.raises(ValueError, match="unsafe transport"): + ServerAgentConfig.from_managed_payload(policy) + + +def test_system_context_reports_host_and_available_commands(tmp_path): + config = AgentConfig( + workspace_root=tmp_path, + trace_path=tmp_path / "trace.jsonl", + session_id="session-1", + ) + config = replace( + config, + commands=replace(config.commands, allowlisted_commands=["git", "missing"]), + ) + adapter = object.__new__(OpenAICompatibleReactAdapter) + adapter._config = config + + with patch("code4me2_agent.adapters.platform.system", return_value="Windows"), patch( + "code4me2_agent.command_tools.available_commands", return_value=["git"] + ): + context = adapter._system_context() + + assert "operating system is Windows" in context + assert "executable commands allowed by policy are: git" in context + assert "Never assume Bash" in context + + +def test_managed_telemetry_does_not_write_project_trace(tmp_path): + trace_path = tmp_path / ".code4me" / "acp-trace.jsonl" + config = AgentConfig( + workspace_root=tmp_path, + trace_path=trace_path, + session_id="session-1", + managed_mode=True, + ) + recorder = AgentTelemetryRecorder(config, sinks=[]) + + recorder.record( + event_type="agent.request.received", + run_id="run-1", + request_id="request-1", + parent_event_id=None, + payload={"content": "private participant prompt"}, + contains_user_prompt=True, + ) + + assert not trace_path.exists() + + +def test_content_preference_redacts_before_telemetry_sink(tmp_path): + captured = [] + + class CaptureSink: + def append(self, event): + captured.append(event) + + config = AgentConfig( + workspace_root=tmp_path, + trace_path=tmp_path / "trace.jsonl", + session_id="session-1", + managed_mode=True, + store_agent_content=False, + ) + recorder = AgentTelemetryRecorder(config, sinks=[CaptureSink()]) + + recorder.record( + event_type="agent.tool.completed", + run_id="run-1", + request_id="request-1", + parent_event_id=None, + payload={ + "tool_name": "run_command", + "status": "completed", + "argv": ["cat", "secret.txt"], + "stdout": "private contents", + "exit_code": 0, + }, + raw_payload={"content": "private contents"}, + ) + + assert captured[0]["payload"] == { + "tool_name": "run_command", + "status": "completed", + "exit_code": 0, + } + assert captured[0]["raw_payload"] is None From a84c355c592194b62b4d20bc8d59c662175b8871 Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Fri, 11 Sep 2026 12:36:27 +0300 Subject: [PATCH 010/115] chore: runtime packaging, docs and dev compose defaults --- .github/workflows/build-managed-runtime.yml | 190 ++++++++++++++++++++ .gitignore | 27 ++- docker-compose.dev-arm.yaml | 92 +++++----- docker-compose.yml | 6 + docs/MANAGED_AGENT_DEPLOYMENT.md | 25 +++ packaging/README.md | 50 ++++++ packaging/archive_runtime.py | 67 +++++++ packaging/code4me2-agent.spec | 37 ++++ packaging/create_release_manifest.py | 51 ++++++ packaging/requirements-build.lock | 5 + packaging/requirements-runtime.lock | 114 ++++++++++++ packaging/requirements-test.lock | 8 + packaging/stamp_runtime_version.py | 34 ++++ 13 files changed, 658 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/build-managed-runtime.yml create mode 100644 docs/MANAGED_AGENT_DEPLOYMENT.md create mode 100644 packaging/README.md create mode 100644 packaging/archive_runtime.py create mode 100644 packaging/code4me2-agent.spec create mode 100644 packaging/create_release_manifest.py create mode 100644 packaging/requirements-build.lock create mode 100644 packaging/requirements-runtime.lock create mode 100644 packaging/requirements-test.lock create mode 100644 packaging/stamp_runtime_version.py diff --git a/.github/workflows/build-managed-runtime.yml b/.github/workflows/build-managed-runtime.yml new file mode 100644 index 0000000..452644a --- /dev/null +++ b/.github/workflows/build-managed-runtime.yml @@ -0,0 +1,190 @@ +name: Build managed runtime + +on: + workflow_dispatch: + inputs: + version: + description: Runtime version to stamp into the manifest + required: true + default: 0.1.0 + publishRelease: + description: Publish immutable runtime-v release assets + type: boolean + required: true + default: false + push: + paths: + - src/code4me2_agent/** + - packaging/** + - pyproject.toml + - tests/code4me2_agent/** + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - os: macos-15 + platform: macos-arm64 + executable: dist/code4me2-agent/code4me2-agent + - os: macos-15-intel + platform: macos-x64 + executable: dist/code4me2-agent/code4me2-agent + - os: windows-2022 + platform: windows-x64 + executable: dist/code4me2-agent/code4me2-agent.exe + - os: ubuntu-24.04 + platform: linux-x64 + executable: dist/code4me2-agent/code4me2-agent + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Verify native runner architecture + shell: bash + env: + EXPECTED_PLATFORM: ${{ matrix.platform }} + run: | + python - <<'PY' + import os + import platform + + expected = os.environ["EXPECTED_PLATFORM"].rsplit("-", 1)[-1] + actual = platform.machine().lower() + normalized = "arm64" if actual in {"arm64", "aarch64"} else "x64" if actual in {"x86_64", "amd64"} else actual + if normalized != expected: + raise SystemExit(f"runner architecture is {normalized!r}, expected {expected!r}") + PY + - name: Stamp runtime version + shell: bash + run: | + version="${{ inputs.version }}" + if [ -z "$version" ]; then version="0.1.0-dev.${GITHUB_SHA:0:12}"; fi + python packaging/stamp_runtime_version.py --version "$version" + - name: Install locked runtime and packaging toolchain + shell: bash + run: | + python -m pip install \ + -r packaging/requirements-runtime.lock \ + -r packaging/requirements-build.lock + python -m pip install --no-deps . + - name: Run managed runtime tests + shell: bash + env: + PYTHONPATH: ${{ github.workspace }}/src + run: | + python -m pip install -r packaging/requirements-test.lock + python -m pytest -q tests/code4me2_agent + - name: Build one-folder bundle + run: python -m PyInstaller --clean --noconfirm packaging/code4me2-agent.spec + - name: Sign macOS runtime + if: github.event_name == 'workflow_dispatch' && inputs.publishRelease && startsWith(matrix.platform, 'macos') + shell: bash + env: + CERTIFICATE_P12: ${{ secrets.CODE4ME_MACOS_CERTIFICATE_P12_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.CODE4ME_MACOS_CERTIFICATE_PASSWORD }} + SIGNING_IDENTITY: ${{ secrets.CODE4ME_MACOS_SIGNING_IDENTITY }} + run: | + test -n "$CERTIFICATE_P12" && test -n "$CERTIFICATE_PASSWORD" && test -n "$SIGNING_IDENTITY" + keychain="$RUNNER_TEMP/code4me-signing.keychain-db" + certificate="$RUNNER_TEMP/code4me-signing.p12" + printf '%s' "$CERTIFICATE_P12" | base64 -D > "$certificate" + security create-keychain -p temporary "$keychain" + security set-keychain-settings -lut 21600 "$keychain" + security unlock-keychain -p temporary "$keychain" + security import "$certificate" -P "$CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$keychain" + security set-key-partition-list -S apple-tool:,apple: -s -k temporary "$keychain" + security list-keychains -d user -s "$keychain" login.keychain-db + signed=0 + while IFS= read -r binary; do + if file "$binary" | grep -q 'Mach-O'; then + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$binary" + codesign --verify --strict --verbose=2 "$binary" + signed=$((signed + 1)) + fi + done < <(find dist/code4me2-agent -depth -type f) + test "$signed" -gt 0 + - name: Sign Windows runtime + if: github.event_name == 'workflow_dispatch' && inputs.publishRelease && startsWith(matrix.platform, 'windows') + shell: pwsh + env: + CERTIFICATE_PFX: ${{ secrets.CODE4ME_WINDOWS_CERTIFICATE_PFX_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.CODE4ME_WINDOWS_CERTIFICATE_PASSWORD }} + run: | + if (-not $env:CERTIFICATE_PFX -or -not $env:CERTIFICATE_PASSWORD) { throw "Windows signing secrets are required" } + $certificatePath = Join-Path $env:RUNNER_TEMP "code4me-signing.pfx" + [IO.File]::WriteAllBytes($certificatePath, [Convert]::FromBase64String($env:CERTIFICATE_PFX)) + $password = ConvertTo-SecureString $env:CERTIFICATE_PASSWORD -AsPlainText -Force + $certificate = Import-PfxCertificate -FilePath $certificatePath -CertStoreLocation Cert:\CurrentUser\My -Password $password + $signTool = Get-ChildItem "${env:ProgramFiles(x86)}\Windows Kits\10\bin" -Recurse -Filter signtool.exe | + Where-Object FullName -Match '\\x64\\signtool.exe$' | Sort-Object FullName -Descending | Select-Object -First 1 + if (-not $signTool) { throw "signtool.exe was not found" } + Get-ChildItem dist\code4me2-agent -Recurse -File | Where-Object Extension -In '.exe', '.dll' | ForEach-Object { + & $signTool.FullName sign /fd SHA256 /td SHA256 /tr http://timestamp.digicert.com /sha1 $certificate.Thumbprint $_.FullName + if ($LASTEXITCODE -ne 0) { throw "Signing failed for $($_.FullName)" } + } + - name: Self-check + shell: bash + run: | + expected="${{ inputs.version }}" + if [ -z "$expected" ]; then expected="0.1.0-dev.${GITHUB_SHA:0:12}"; fi + actual=$("${{ matrix.executable }}" --version) + test "$actual" = "$expected" + "${{ matrix.executable }}" --self-check + # Archive layout contract (must match ManagedRuntimeInstaller): + # flat zip, executable at the archive root, macOS framework symlinks + # materialized as real files (the installer's ZipInputStream extraction + # does not restore symlinks). + - name: Archive native bundle + run: python packaging/archive_runtime.py --platform "${{ matrix.platform }}" + - name: Notarize macOS runtime archive + if: github.event_name == 'workflow_dispatch' && inputs.publishRelease && startsWith(matrix.platform, 'macos') + shell: bash + env: + APPLE_ID: ${{ secrets.CODE4ME_APPLE_ID }} + APPLE_TEAM_ID: ${{ secrets.CODE4ME_APPLE_TEAM_ID }} + APPLE_APP_PASSWORD: ${{ secrets.CODE4ME_APPLE_APP_PASSWORD }} + run: | + test -n "$APPLE_ID" && test -n "$APPLE_TEAM_ID" && test -n "$APPLE_APP_PASSWORD" + xcrun notarytool submit "code4me-agent-${{ matrix.platform }}.zip" \ + --apple-id "$APPLE_ID" \ + --team-id "$APPLE_TEAM_ID" \ + --password "$APPLE_APP_PASSWORD" \ + --wait + - uses: actions/upload-artifact@v4 + with: + name: code4me-agent-${{ matrix.platform }} + path: code4me-agent-${{ matrix.platform }}.zip + if-no-files-found: error + + publish: + if: github.event_name == 'workflow_dispatch' && inputs.publishRelease + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: code4me-agent-* + path: release-assets + merge-multiple: true + - name: Create pinned release metadata + run: | + python packaging/create_release_manifest.py \ + --directory release-assets \ + --version "${{ inputs.version }}" \ + --server-commit "$GITHUB_SHA" \ + --output release-assets/code4me-managed-runtime-release.json + - name: Publish immutable runtime release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "runtime-v${{ inputs.version }}" \ + --target "$GITHUB_SHA" \ + --title "Code4Me managed runtime ${{ inputs.version }}" \ + release-assets/* diff --git a/.gitignore b/.gitignore index 898d4e0..1fc221c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,25 @@ .DS_Store .env -node_modules -.venv +.env.* +!.env.example +node_modules/ +.venv/ *.egg-info/ -.idea -secrets +.idea/ +secrets/ **/__pycache__/ -.hypothesis -.cache -data +.hypothesis/ +.cache/ +data/ .coverage + +# Native/PyInstaller and package build output +build/ +dist/ +*.py[cod] + +# Test, lint, type-check and coverage caches +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +htmlcov/ diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml index 0d7ce3a..58f1299 100644 --- a/docker-compose.dev-arm.yaml +++ b/docker-compose.dev-arm.yaml @@ -15,20 +15,20 @@ services: image: pgvector/pgvector:pg16 container_name: postgres ports: - - ${DB_PORT}:${DB_PORT} + - ${DB_PORT:-2345}:${DB_PORT:-2345} expose: - - ${DB_PORT} + - ${DB_PORT:-2345} volumes: - .:/app - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql environment: - - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_USER=${DB_USER} - - POSTGRES_DB=${DB_NAME} - - PGPORT=${DB_PORT} - command: ["postgres", "-p", "${DB_PORT}"] + - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres} + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_DB=${DB_NAME:-code4meV2} + - PGPORT=${DB_PORT:-2345} + command: ["postgres", "-p", "${DB_PORT:-2345}"] healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -p ${DB_PORT}"] + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -p ${DB_PORT:-2345}"] start_period: 0s start_interval: 0s interval: 60s @@ -39,15 +39,15 @@ services: image: redis:7.0 container_name: redis ports: - - ${REDIS_PORT}:${REDIS_PORT} + - ${REDIS_PORT:-6379}:${REDIS_PORT:-6379} expose: - - ${REDIS_PORT} + - ${REDIS_PORT:-6379} volumes: - .:/app - ./redis.conf:/usr/local/etc/redis/redis.conf - command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--port", "${REDIS_PORT}"] + command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--port", "${REDIS_PORT:-6379}"] healthcheck: - test: ["CMD", "redis-cli", "-p", "${REDIS_PORT}", "ping"] + test: ["CMD", "redis-cli", "-p", "${REDIS_PORT:-6379}", "ping"] start_period: 0s start_interval: 0s interval: 60s @@ -58,10 +58,10 @@ services: image: redis:7.0 container_name: redis-celery ports: - - ${CELERY_BROKER_PORT}:${CELERY_BROKER_PORT} + - ${CELERY_BROKER_PORT:-6380}:${CELERY_BROKER_PORT:-6380} expose: - - ${CELERY_BROKER_PORT} - command: ["redis-server", "--port", "${CELERY_BROKER_PORT}"] + - ${CELERY_BROKER_PORT:-6380} + command: ["redis-server", "--port", "${CELERY_BROKER_PORT:-6380}"] volumes: - .:/app healthcheck: @@ -77,6 +77,8 @@ services: context: . dockerfile: Dockerfile.cpu container_name: celery-worker + env_file: + - ./src/backend/completion/.env command: [ "/bin/bash", "-c", @@ -89,17 +91,20 @@ services: condition: service_healthy environment: - REDIS_HOST=redis - - REDIS_PORT=${REDIS_PORT} + - REDIS_PORT=${REDIS_PORT:-6379} - CELERY_BROKER_HOST=redis-celery - - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT:-6380} - DB_HOST=db - - DB_PORT=${DB_PORT} + - DB_PORT=${DB_PORT:-2345} + - DB_NAME=${DB_NAME:-code4meV2} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD:-postgres} - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db - PYTHONPATH=/app/src - PYTHONMULTIPROCESSING_START_METHOD=spawn - TOKENIZERS_PARALLELISM=false - OMP_NUM_THREADS=1 - - HF_TOKEN=${HF_TOKEN} + - HF_TOKEN=${HF_TOKEN:-} - HF_HOME=/app/hf_cache - HF_HUB_CACHE=/app/hf_cache - TRANSFORMERS_CACHE=/app/hf_cache @@ -108,7 +113,7 @@ services: - PRELOAD_MODELS=false volumes: - .:/app - - ${DATA_DIR}/hf:/app/hf_cache + - ${DATA_DIR:-./data}/hf:/app/hf_cache healthcheck: test: [ @@ -127,40 +132,45 @@ services: context: . dockerfile: Dockerfile.cpu container_name: backend + env_file: + - ./src/backend/completion/.env command: ["/bin/bash", "-c", "source activate myenv && python /app/src/main.py"] expose: - - ${SERVER_PORT} + - ${SERVER_PORT:-8008} ports: - - "${SERVER_PORT}:8008" + - "${SERVER_PORT:-8008}:8008" volumes: - .:/app - - ${DATA_DIR}/hf:/app/hf_cache + - ${DATA_DIR:-./data}/hf:/app/hf_cache environment: - - SERVER_PORT=${SERVER_PORT} - - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} + - SERVER_PORT=${SERVER_PORT:-8008} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-} - REDIS_HOST=redis - - REDIS_PORT=${REDIS_PORT} + - REDIS_PORT=${REDIS_PORT:-6379} - CELERY_BROKER_HOST=redis-celery - - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT:-6380} - DB_HOST=db - - DB_PORT=${DB_PORT} + - DB_PORT=${DB_PORT:-2345} + - DB_NAME=${DB_NAME:-code4meV2} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD:-postgres} - WEBSITE_HOST=website - - WEBSITE_PORT=${WEBSITE_PORT} + - WEBSITE_PORT=${WEBSITE_PORT:-3000} - PYTHONPATH=/app/src - - HF_TOKEN=${HF_TOKEN} + - HF_TOKEN=${HF_TOKEN:-} - HF_HOME=/app/hf_cache - HF_HUB_CACHE=/app/hf_cache - TRANSFORMERS_CACHE=/app/hf_cache - MODEL_CACHE_DIR=/app/hf_cache - PRELOAD_MODELS=false # Agent subsystem (ACP) upstream provider config, see .env.example - - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL} - - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY} - - OLLAMA_API_KEY=${OLLAMA_API_KEY} - - OPENAI_API_KEY=${OPENAI_API_KEY} - - GROQ_API_KEY=${GROQ_API_KEY} - - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} - - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT} + - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL:-http://host.docker.internal:11434/v1} + - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY:-} + - OLLAMA_API_KEY=${OLLAMA_API_KEY:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - GROQ_API_KEY=${GROQ_API_KEY:-} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} + - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT:-} # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an # Ollama instance running on the host from inside this container. extra_hosts: @@ -183,18 +193,18 @@ services: image: node:18 container_name: website working_dir: /website - command: bash -c "npm install && npm start -- --port ${WEBSITE_PORT}" + command: bash -c "npm install && npm start -- --port ${WEBSITE_PORT:-3000}" expose: - - ${WEBSITE_PORT} + - ${WEBSITE_PORT:-3000} ports: - - "${WEBSITE_PORT}:${WEBSITE_PORT}" + - "${WEBSITE_PORT:-3000}:${WEBSITE_PORT:-3000}" volumes: - ./src/website:/website depends_on: backend: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--spider", "http://localhost:${WEBSITE_PORT}"] + test: ["CMD", "wget", "--spider", "http://localhost:${WEBSITE_PORT:-3000}"] start_period: 0s interval: 60s timeout: 5s diff --git a/docker-compose.yml b/docker-compose.yml index b3bac15..e476b6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,9 @@ services: - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db - PYTHONPATH=/app/src - PYTHONMULTIPROCESSING_START_METHOD=spawn @@ -188,6 +191,9 @@ services: - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} - WEBSITE_HOST=website - WEBSITE_PORT=${WEBSITE_PORT} - PYTHONPATH=/app/src diff --git a/docs/MANAGED_AGENT_DEPLOYMENT.md b/docs/MANAGED_AGENT_DEPLOYMENT.md new file mode 100644 index 0000000..b7cfdbb --- /dev/null +++ b/docs/MANAGED_AGENT_DEPLOYMENT.md @@ -0,0 +1,25 @@ +# Managed agent deployment + +The participant plugin assumes a globally reachable HTTPS backend and managed ACP protocol v1. Provider credentials belong only in the backend environment; participant machines must not receive them. + +## Deployment order + +1. Back up the study database. +2. Deploy the candidate backend with traffic disabled or readiness failing. +3. Run `python src/database/migration/migration_manager.py migrate` as a one-shot deployment job. In the Compose image, use `/opt/conda/envs/myenv/bin/python src/database/migration/migration_manager.py migrate`; the command now loads the same non-overriding `.env` fallback as the application and exits unsuccessfully unless the database reaches the code's exact Alembic head. +4. Confirm the database is at the expected Alembic head and the managed ACP capabilities endpoint reports ready. +5. Run the managed agent smoke test through the public HTTPS origin. +6. Enable traffic, then distribute the plugin ZIP built for this protocol and backend version. + +Do not reset a deployed study database during an upgrade. A failed migration must stop deployment and leave the existing application version serving until the database is restored or the migration is corrected. + +## Release checks + +- Upgrade a copy of the previous deployed schema and initialize an empty database. +- Confirm the active study contains only runtime profiles certified for the participant release. +- Confirm the backend holds every provider key named by an active profile. +- Verify grant exchange, run creation, inference, telemetry, logout revocation, and expiry recovery. +- Block direct provider access on the participant test machine and confirm inference still succeeds. +- Scan the plugin and runtime artifacts for `.env` files, provider credentials, source-checkout paths, and `local-dev` references. + +Record the server commit, Alembic head, plugin commit, runtime version, runtime checksums, IDE version, and tested operating systems with each release. diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 0000000..1299e8b --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,50 @@ +# Managed runtime packaging + +Build on each target operating system and architecture; PyInstaller does not +cross-compile. From the repository root: + +```bash +python packaging/stamp_runtime_version.py --version 1.2.0 +python -m pip install -r packaging/requirements-runtime.lock -r packaging/requirements-build.lock +python -m pip install -r packaging/requirements-test.lock +python -m pip install --no-deps . +PYTHONPATH=src python -m pytest tests/agent_runtime +python -m PyInstaller --clean --noconfirm packaging/code4me2-agent.spec +dist/code4me2-agent/code4me2-agent --self-check +``` + +The version is embedded in the executable and must match the immutable runtime +release tag and plugin manifest. The workflow performs this stamp automatically; +reset `_build_version.py` to its source-development value after a manual build. + +`requirements-runtime.lock` pins the complete participant dependency graph. +Regenerate it with the command recorded in the lock file and review all changes +before publishing a study release. + +The one-folder artifact is `dist/code4me2-agent/` (the executable is suffixed +`.exe` on Windows). Archive the directory *contents* as a flat zip: the +executable must sit at the archive root (`code4me2-agent` / +`code4me2-agent.exe`), with `_internal/` beside it. Materialize macOS +framework symlinks as real files under their link paths (the plugin installer +extracts with `ZipInputStream` and does not restore symlinks); the extracted +copy must still pass `--self-check`. The plugin release manifest supplies its +version, target and SHA-256 digest and embeds one archive per supported +target. The `build-managed-runtime` workflow implements this layout; do not +revert to archiving the parent folder. + +macOS builds must be signed and notarized after collection. Windows builds +must be Authenticode-signed after collection. Perform both operations in their +native release jobs and test the final signed archive rather than this raw +directory. Publishing from the workflow requires the `CODE4ME_MACOS_*`, +`CODE4ME_APPLE_*`, and `CODE4ME_WINDOWS_*` signing secrets named in the +workflow; missing secrets fail the release rather than publishing unsigned +participant binaries. + +## Deferred managed runtimes + +- TODO: package, register, repair and certify Goose without PATH discovery. +- TODO: package, register, repair and certify Codex without Node/npm or a source checkout. +- TODO: add both runtimes to shared onboarding, policy enforcement and telemetry tests. + +Existing Goose and Codex developer integrations remain supported while these +participant distribution tasks are pending. diff --git a/packaging/archive_runtime.py b/packaging/archive_runtime.py new file mode 100644 index 0000000..32276e5 --- /dev/null +++ b/packaging/archive_runtime.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Create the flat, symlink-free runtime ZIP consumed by the IDE plugin.""" + +from __future__ import annotations + +import argparse +import shutil +import stat +import zipfile +from pathlib import Path + + +def collect_tree(root: Path) -> dict[str, Path]: + entries: dict[str, Path] = {} + root_real = root.resolve() + + def visit(directory: Path, archive_prefix: Path, ancestors: frozenset[Path]) -> None: + real_directory = directory.resolve() + if real_directory in ancestors: + return + next_ancestors = ancestors | {real_directory} + for child in sorted(directory.iterdir(), key=lambda item: item.name): + archive_path = (archive_prefix / child.name).as_posix() + resolved_child = child.resolve() + if not resolved_child.is_relative_to(root_real): + raise SystemExit( + f"runtime bundle contains a link outside its root: {child}" + ) + if child.is_dir(): + visit(child, archive_prefix / child.name, next_ancestors) + elif child.is_file(): + entries.setdefault(archive_path, resolved_child) + + visit(root, Path(), frozenset()) + return entries + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path("dist/code4me2-agent")) + parser.add_argument("--platform", required=True) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + root = args.root.resolve() + if not root.is_dir(): + raise SystemExit(f"runtime bundle does not exist: {root}") + executable = "code4me2-agent.exe" if args.platform.startswith("windows") else "code4me2-agent" + entries = collect_tree(root) + if executable not in entries: + raise SystemExit(f"runtime executable is missing at archive root: {executable}") + + output = args.output or Path(f"code4me-agent-{args.platform}.zip") + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as archive: + for archive_path, source in sorted(entries.items()): + info = zipfile.ZipInfo(archive_path) + mode = 0o755 if archive_path == executable else 0o644 + info.external_attr = (stat.S_IFREG | mode) << 16 + info.create_system = 3 + info.compress_type = zipfile.ZIP_DEFLATED + with source.open("rb") as source_handle, archive.open(info, "w") as target_handle: + shutil.copyfileobj(source_handle, target_handle, length=1024 * 1024) + print(f"archived {output} ({len(entries)} files)") + + +if __name__ == "__main__": + main() diff --git a/packaging/code4me2-agent.spec b/packaging/code4me2-agent.spec new file mode 100644 index 0000000..507f794 --- /dev/null +++ b/packaging/code4me2-agent.spec @@ -0,0 +1,37 @@ +from pathlib import Path + +# SPECPATH is the directory containing this spec file (packaging/), +# so its parent is the server repository root. +project_root = Path(SPECPATH).parent +source_root = project_root / "src" + +analysis = Analysis( + [str(source_root / "code4me2_agent" / "cli.py")], + pathex=[str(source_root)], + binaries=[], + datas=[], + hiddenimports=[], + excludes=["backend", "database", "celery", "torch", "transformers"], + noarchive=False, +) + +pyz = PYZ(analysis.pure) + +executable = EXE( + pyz, + analysis.scripts, + [], + exclude_binaries=True, + name="code4me2-agent", + console=True, + disable_windowed_traceback=False, +) + +collection = COLLECT( + executable, + analysis.binaries, + analysis.datas, + strip=False, + upx=False, + name="code4me2-agent", +) diff --git a/packaging/create_release_manifest.py b/packaging/create_release_manifest.py new file mode 100644 index 0000000..0cda86b --- /dev/null +++ b/packaging/create_release_manifest.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Create checksummed metadata that pins native bundles to a server commit.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--server-commit", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + archives = sorted(args.directory.glob("code4me-agent-*.zip")) + expected = { + "code4me-agent-macos-arm64.zip", + "code4me-agent-macos-x64.zip", + "code4me-agent-windows-x64.zip", + "code4me-agent-linux-x64.zip", + } + names = {archive.name for archive in archives} + if names != expected: + raise SystemExit(f"runtime release is incomplete: expected {sorted(expected)}, found {sorted(names)}") + payload = { + "manifest_version": 1, + "managed_protocol_version": "1", + "runtime_version": args.version, + "server_commit": args.server_commit, + "artifacts": [ + {"archive": archive.name, "sha256": sha256(archive)} for archive in archives + ], + } + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/packaging/requirements-build.lock b/packaging/requirements-build.lock new file mode 100644 index 0000000..771c720 --- /dev/null +++ b/packaging/requirements-build.lock @@ -0,0 +1,5 @@ +# Install the project itself first so its runtime dependencies are resolved: +# python -m pip install . -r packaging/requirements-build.lock +pyinstaller==6.16.0 +setuptools==80.9.0 +wheel==0.45.1 diff --git a/packaging/requirements-runtime.lock b/packaging/requirements-runtime.lock new file mode 100644 index 0000000..fbb13c7 --- /dev/null +++ b/packaging/requirements-runtime.lock @@ -0,0 +1,114 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --python-version 3.13 --output-file packaging/requirements-runtime.lock +agent-client-protocol==0.12.1 + # via code4me2-agent (pyproject.toml) +annotated-types==0.8.0 + # via pydantic +anyio==4.15.1 + # via + # httpx + # httpx2 + # mcp + # openai + # sse-starlette + # starlette +attrs==26.1.0 + # via + # jsonschema + # referencing +certifi==2026.7.22 + # via + # httpcore + # httpx +cffi==2.1.1 + # via cryptography +click==8.5.0 + # via uvicorn +cryptography==50.0.1 + # via pyjwt +distro==1.9.0 + # via openai +h11==0.16.0 + # via + # httpcore + # httpcore2 + # uvicorn +httpcore==1.0.9 + # via httpx +httpcore2==2.12.0 + # via httpx2 +httpx==0.28.1 + # via openai +httpx2==2.12.0 + # via mcp +idna==3.19 + # via + # anyio + # httpx + # httpx2 +jiter==0.16.0 + # via openai +jsonschema==4.26.0 + # via mcp +jsonschema-specifications==2025.9.1 + # via jsonschema +mcp==2.2.0 + # via code4me2-agent (pyproject.toml) +mcp-types==2.2.0 + # via mcp +openai==1.109.1 + # via code4me2-agent (pyproject.toml) +opentelemetry-api==1.44.0 + # via mcp +pycparser==3.0 + # via cffi +pydantic==2.13.5 + # via + # agent-client-protocol + # mcp + # mcp-types + # openai +pydantic-core==2.46.5 + # via pydantic +pyjwt==2.13.0 + # via mcp +python-multipart==0.0.32 + # via mcp +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +rpds-py==2026.6.3 + # via + # jsonschema + # referencing +sniffio==1.3.1 + # via openai +sse-starlette==3.4.11 + # via mcp +starlette==1.6.0 + # via + # mcp + # sse-starlette +tqdm==4.70.0 + # via openai +truststore==0.10.4 + # via + # httpcore2 + # httpx2 +typing-extensions==4.16.0 + # via + # anyio + # mcp + # mcp-types + # openai + # opentelemetry-api + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # mcp + # pydantic +uvicorn==0.52.4 + # via mcp diff --git a/packaging/requirements-test.lock b/packaging/requirements-test.lock new file mode 100644 index 0000000..35cd878 --- /dev/null +++ b/packaging/requirements-test.lock @@ -0,0 +1,8 @@ +# Runtime release smoke-test dependencies. Keep fully pinned for native builds. +colorama==0.4.6 ; sys_platform == "win32" +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +pygments==2.21.0 +pytest==9.1.1 +pytest-env==1.1.5 diff --git a/packaging/stamp_runtime_version.py b/packaging/stamp_runtime_version.py new file mode 100644 index 0000000..4e4a7b7 --- /dev/null +++ b/packaging/stamp_runtime_version.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Stamp the immutable version reported by a packaged runtime binary.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +SEMVER = re.compile( + r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\Z" +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--version", required=True) + parser.add_argument( + "--output", + type=Path, + default=Path("src/code4me2_agent/_build_version.py"), + ) + args = parser.parse_args() + if not SEMVER.fullmatch(args.version): + raise SystemExit(f"runtime version must be SemVer: {args.version!r}") + args.output.write_text( + '"""Generated by packaging/stamp_runtime_version.py."""\n\n' + f'RUNTIME_VERSION = "{args.version}"\n', + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() From 28840d599b951d11ec750e3f0bee968d9cf5da1b Mon Sep 17 00:00:00 2001 From: myildirim19 Date: Fri, 11 Sep 2026 14:38:07 +0300 Subject: [PATCH 011/115] chore:temp version file --- src/code4me2_agent/_build_version.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/code4me2_agent/_build_version.py diff --git a/src/code4me2_agent/_build_version.py b/src/code4me2_agent/_build_version.py new file mode 100644 index 0000000..9949233 --- /dev/null +++ b/src/code4me2_agent/_build_version.py @@ -0,0 +1,7 @@ +"""Runtime version used for source checkouts. + +Release and local ZIP builds temporarily replace this value with their immutable +artifact version via ``packaging/stamp_runtime_version.py``. +""" + +RUNTIME_VERSION = "0.0.0-dev" From ec9d04a67dfd2fa167665fd80987822eae0af2c5 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Fri, 11 Sep 2026 18:14:42 +0200 Subject: [PATCH 012/115] fix: raise err on email delivery failure --- src/celery_app/tasks/db_tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/celery_app/tasks/db_tasks.py b/src/celery_app/tasks/db_tasks.py index 2565ad3..dc67dad 100644 --- a/src/celery_app/tasks/db_tasks.py +++ b/src/celery_app/tasks/db_tasks.py @@ -288,8 +288,10 @@ def send_verification_email_task(user_id: str, user_email: str, user_name: str): force_reset_exp=True, ) - # Send the verification email with the generated token - send_verification_email(user_email, user_name, verification_token) + # Propagate delivery failures so Celery marks the job as failed instead of + # reporting a false success to operators. + if not send_verification_email(user_email, user_name, verification_token): + raise RuntimeError(f"Verification email delivery failed for user {user_id}") @celery.task From bfde7da8f11cd369f7cc693429424e483346786b Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Fri, 11 Sep 2026 18:15:16 +0200 Subject: [PATCH 013/115] fix: use legacy mode for MCP tool clients --- src/code4me2_agent/mcp_tools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/code4me2_agent/mcp_tools.py b/src/code4me2_agent/mcp_tools.py index e506283..bd2c0de 100644 --- a/src/code4me2_agent/mcp_tools.py +++ b/src/code4me2_agent/mcp_tools.py @@ -33,6 +33,10 @@ class _McpTool: definition: dict[str, Any] +def _legacy_mcp_client(parameters: StdioServerParameters) -> Client: + return Client(parameters, mode="legacy") + + class StdioMcpToolBroker: """Own stdio MCP clients and expose their tools to the synchronous agent loop. @@ -47,7 +51,7 @@ def __init__( servers: list[object], *, cwd: Path, - client_factory: Callable[[StdioServerParameters], Client] = Client, + client_factory: Callable[[StdioServerParameters], Client] = _legacy_mcp_client, ) -> None: self._servers = list(servers) self._cwd = cwd @@ -71,7 +75,7 @@ def open( servers: list[object], *, cwd: Path, - client_factory: Callable[[StdioServerParameters], Client] = Client, + client_factory: Callable[[StdioServerParameters], Client] = _legacy_mcp_client, startup_timeout_seconds: float = _MCP_STARTUP_TIMEOUT_SECONDS, ) -> StdioMcpToolBroker | None: if not servers: From 54f9a21876fff057fb5918b81fc9f80f9acb0d54 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Fri, 11 Sep 2026 18:15:55 +0200 Subject: [PATCH 014/115] fix: use correct .env --- docker-compose.dev-arm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml index 58f1299..63c037d 100644 --- a/docker-compose.dev-arm.yaml +++ b/docker-compose.dev-arm.yaml @@ -78,7 +78,7 @@ services: dockerfile: Dockerfile.cpu container_name: celery-worker env_file: - - ./src/backend/completion/.env + - ./.env command: [ "/bin/bash", "-c", @@ -133,7 +133,7 @@ services: dockerfile: Dockerfile.cpu container_name: backend env_file: - - ./src/backend/completion/.env + - ./.env command: ["/bin/bash", "-c", "source activate myenv && python /app/src/main.py"] expose: - ${SERVER_PORT:-8008} From 959397fdb55bf6b62443aaed4b7fe8aad076c2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cem=20g=C3=BCng=C3=B6r?= <77344358+cemgungor1@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:03:30 +0200 Subject: [PATCH 015/115] Feat/ab study agent assignments (#3) * feat: add study-scoped agent assignments * feat: manage agent arms and assignments by study --- src/agents/event_writer.py | 12 +- src/agents/ingest.py | 48 ++- src/agents/lifecycle.py | 2 +- src/agents/registry.py | 94 +++-- src/backend/routers/acp/__init__.py | 13 +- src/backend/routers/agent/ingest.py | 12 +- src/backend/routers/agent/profiles.py | 142 ++++++-- src/backend/routers/agents.py | 25 +- src/backend/routers/analytics/studies.py | 96 +++-- src/database/crud.py | 280 +++++++++++++-- src/database/db_schemas.py | 67 ++++ ...f7a9_add_experiment_execution_integrity.py | 328 ++++++++++++++++++ .../components/analytics/StudyManagement.css | 63 ++++ .../components/analytics/StudyManagement.js | 165 ++++++++- src/website/src/pages/AgentAssignments.js | 209 +++++++---- src/website/src/pages/AgentProfiles.js | 9 +- src/website/src/utils/api.js | 32 +- 17 files changed, 1313 insertions(+), 284 deletions(-) create mode 100644 src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py diff --git a/src/agents/event_writer.py b/src/agents/event_writer.py index b8c64fa..9b0f37f 100644 --- a/src/agents/event_writer.py +++ b/src/agents/event_writer.py @@ -53,13 +53,16 @@ def write_model_call_event( parent_span_id = span.get("parent_span_id") db = app.get_db_session() try: - event_index = crud.count_agent_events_for_task(db, task_uuid) + event_index = crud.reserve_agent_event_indexes(db, task_uuid, 1) + source_event_id = span.get("span_id") or record.request_id crud.append_agent_event( db, task_id=task_uuid, event_index=event_index, event_type="model_call", source=SOURCE_PROXY, + source_event_id=source_event_id, + ignore_duplicate_source=source_event_id is not None, latency_ms=latency_ms, # span identifiers span_id=span.get("span_id") or record.request_id, @@ -147,7 +150,7 @@ def write_tool_call_events( db = app.get_db_session() try: - event_index = crud.count_agent_events_for_task(db, task_uuid) + event_index = crud.reserve_agent_event_indexes(db, task_uuid, len(tool_executions)) for offset, tc in enumerate(tool_executions): arguments = tc.get("arguments") result = tc.get("result") @@ -165,14 +168,17 @@ def write_tool_call_events( if isinstance(result, str): tool_result_length = len(result) + source_event_id = tc.get("id") or tc.get("tool_call_id") crud.append_agent_event( db, task_id=task_uuid, event_index=event_index + offset, event_type="tool_call", source=SOURCE_PROXY, + source_event_id=str(source_event_id) if source_event_id else None, + ignore_duplicate_source=source_event_id is not None, latency_ms=latency_ms, - span_id=str(uuid.uuid4()), + span_id=str(source_event_id) if source_event_id else str(uuid.uuid4()), parent_span_id=parent_span, chat_session_index=chat_session_index, tool_name=tc.get("name"), diff --git a/src/agents/ingest.py b/src/agents/ingest.py index bedaa97..2c69d08 100644 --- a/src/agents/ingest.py +++ b/src/agents/ingest.py @@ -305,8 +305,9 @@ def ingest_event_batch( to retry after a network failure — the uploader has no way to know whether a timed-out POST was applied. - The whole batch commits once: a partial write would leave gaps in - ``event_index``, which downstream step counting relies on being dense. + The whole batch commits once. Source identity makes retries idempotent; + task-local indexes are allocated atomically, so concurrent batches cannot + claim the same event position. """ if not events: return 0, [] @@ -315,34 +316,55 @@ def ingest_event_batch( # even if the batch arrives out of order. ordered = sorted(events, key=lambda e: e.get("sequence") or 0) - span_ids = [str(e["event_id"]) for e in ordered if e.get("event_id")] - already_seen = set(crud.find_existing_agent_event_span_ids(db, span_ids)) + source_event_ids = [str(e["event_id"]) for e in ordered if e.get("event_id")] + already_seen = set( + crud.find_existing_agent_event_source_ids( + db, + task_id=task_id, + source=SOURCE_SELF_REPORT, + source_event_ids=source_event_ids, + ) + ) by_request_id, by_tool_call_id = build_correlation_index(ordered) - next_index = crud.count_agent_events_for_task(db, task_id) - - ingested = 0 skipped: list[str] = [] + pending: list[tuple[dict, str]] = [] + seen_in_batch: set[str] = set() for event in ordered: - span_id = str(event["event_id"]) if event.get("event_id") else None - if span_id is not None and span_id in already_seen: - skipped.append(span_id) + source_event_id = str(event["event_id"]) if event.get("event_id") else None + if source_event_id is None: + continue + if source_event_id in already_seen or source_event_id in seen_in_batch: + skipped.append(source_event_id) continue + seen_in_batch.add(source_event_id) + pending.append((event, source_event_id)) + + first_index = ( + crud.reserve_agent_event_indexes(db, task_id, len(pending)) if pending else 0 + ) + ingested = 0 + for offset, (event, source_event_id) in enumerate(pending): columns = map_event_to_columns( event, content_included=content_included, by_request_id=by_request_id, by_tool_call_id=by_tool_call_id, ) - crud.append_agent_event( + inserted = crud.append_agent_event( db, task_id=task_id, - event_index=next_index + ingested, + event_index=first_index + offset, agent_profile=agent_profile, + source_event_id=source_event_id, + ignore_duplicate_source=True, commit=False, **columns, ) - ingested += 1 + if inserted is None: + skipped.append(source_event_id) + else: + ingested += 1 db.commit() logging.info( diff --git a/src/agents/lifecycle.py b/src/agents/lifecycle.py index 19a9c53..c78553a 100644 --- a/src/agents/lifecycle.py +++ b/src/agents/lifecycle.py @@ -48,7 +48,7 @@ def finalize_agent_task( tool_names = list(event.tool_names_requested) last_event = max(events, key=lambda e: e.event_index) if events else None - total_steps = (last_event.event_index + 1) if last_event is not None else 0 + total_steps = len(events) crud.update_agent_task_status( db, task_id=task_id, diff --git a/src/agents/registry.py b/src/agents/registry.py index 23c4b6e..a4293e5 100644 --- a/src/agents/registry.py +++ b/src/agents/registry.py @@ -5,6 +5,7 @@ import logging import random import uuid +from dataclasses import dataclass from typing import Optional from sqlalchemy.orm import Session @@ -18,46 +19,69 @@ def get_profile(db: Session, name: str) -> Optional[AgentProfile]: return crud.get_agent_profile(db, name) -def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[AgentProfile]: - """Return the agent profile assigned to ``user_id``. +@dataclass(frozen=True) +class AgentAssignmentResolution: + """The selected profile and optional immutable experiment arm for one task.""" - Assignment is server-authoritative and sticky: if the user already has an - assignment (auto or manual) it's returned unchanged, so a participant keeps - the same arm for the life of the experiment regardless of later changes to - the active set. Otherwise a profile is drawn at random (even split) and - persisted as an ``auto`` assignment. + profile: AgentProfile + study_id: Optional[uuid.UUID] = None + assignment_id: Optional[uuid.UUID] = None + arm_name: Optional[str] = None + is_baseline: Optional[bool] = None - The client never picks its own arm — that's the point. A self-selected - bucket would invalidate the comparison. - Candidate pool: when a study is active *and* has selected agent profiles, - the draw is restricted to that study's arms. Otherwise it falls back to all - active profiles, so behaviour is unchanged when no agent study is running - (backwards compatible with the completion-only platform). +def resolve_assignment_context( + db: Session, user_id: uuid.UUID +) -> Optional[AgentAssignmentResolution]: + """Resolve the profile and study assignment for a new task. - Returns ``None`` only when no candidate profiles exist at all, which the - caller surfaces as a 503 — minting a task with no profile would produce - telemetry that can't be attributed to any condition. + An active study owns its participant assignments: one immutable row is + inserted per ``(study_id, user_id)``. The insert is conflict-safe, so + concurrent first contacts converge on the arm selected by the winner. + Outside an agent study there is no experimental assignment; the first active + profile is used deterministically for operational runs. """ - assignment = crud.get_agent_profile_assignment(db, user_id) - if assignment is not None and assignment.profile is not None: - return assignment.profile - - candidates = crud.list_active_study_agent_profiles(db) - pool = "active study" - if not candidates: - candidates = crud.list_active_agent_profiles(db) - pool = "all active profiles" - if not candidates: + active_arm_links = crud.list_active_study_agent_profile_links(db) + if active_arm_links: + study_id = active_arm_links[0].study_id + assignment = crud.get_agent_study_assignment(db, study_id, user_id) + if assignment is None: + chosen_arm = random.choice(active_arm_links) + profile = chosen_arm.profile + if profile is None: + logging.error("[Agent/registry] active study arm has no profile") + return None + assignment = crud.create_agent_study_assignment( + db, + study_id=study_id, + user_id=user_id, + profile_id=profile.profile_id, + arm_name=profile.name, + is_baseline=chosen_arm.is_baseline, + source="auto", + ) + if assignment is None: + assignment = crud.get_agent_study_assignment(db, study_id, user_id) + + if assignment is None or assignment.profile is None: + logging.error("[Agent/registry] could not resolve a study assignment") + return None + return AgentAssignmentResolution( + profile=assignment.profile, + study_id=assignment.study_id, + assignment_id=assignment.assignment_id, + arm_name=assignment.arm_name, + is_baseline=assignment.is_baseline, + ) + + active_profiles = crud.list_active_agent_profiles(db) + if not active_profiles: logging.error("[Agent/registry] no active agent profiles — cannot assign") return None + return AgentAssignmentResolution(profile=active_profiles[0]) - chosen = random.choice(candidates) - crud.set_agent_profile_assignment( - db, user_id=user_id, profile_id=chosen.profile_id, source="auto" - ) - logging.info( - f"[Agent/registry] drew arm {chosen.name!r} for user {str(user_id)[:8]}… " - f"from {pool} ({len(candidates)} candidate(s))" - ) - return chosen + +def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[AgentProfile]: + """Return the profile selected for a new task.""" + resolution = resolve_assignment_context(db, user_id) + return resolution.profile if resolution is not None else None diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py index 059a43a..446e0ee 100644 --- a/src/backend/routers/acp/__init__.py +++ b/src/backend/routers/acp/__init__.py @@ -78,7 +78,7 @@ FALLBACK_MAX_CONTEXT_TOKENS = 16_000 MANAGED_PROTOCOL_VERSION = "1" MANAGED_RUNTIME = "code4me2-agent" -EXPECTED_SCHEMA_REVISION = "f3c4d5e6a7b9" +EXPECTED_SCHEMA_REVISION = "f3c4d5e6f7a9" SUPPORTED_APPROVAL_POLICIES = frozenset({"auto", "per_step", "suggestion_only"}) _BEARER_PREFIX = "Bearer " @@ -668,11 +668,12 @@ def create_managed_run( from agents import registry - profile = registry.resolve_assignment(db, user_id) - if profile is None: + assignment = registry.resolve_assignment_context(db, user_id) + if assignment is None: raise HTTPException( status_code=503, detail="No active study agent profile is assigned to this user" ) + profile = assignment.profile policy = _managed_policy(db, user_id, profile) task = crud.create_agent_task( db, @@ -691,6 +692,12 @@ def create_managed_run( status="running", started_at=datetime.now(timezone.utc), policy_snapshot=policy, + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + study_arm_name=assignment.arm_name, + study_arm_is_baseline=assignment.is_baseline, + consent_content_storage=policy["store_agent_content"], ) return JSONResponse( {"task_id": str(task.task_id), "run_id": body.run_id, "policy": policy}, diff --git a/src/backend/routers/agent/ingest.py b/src/backend/routers/agent/ingest.py index 0a5dd7e..821a641 100644 --- a/src/backend/routers/agent/ingest.py +++ b/src/backend/routers/agent/ingest.py @@ -155,12 +155,14 @@ def _resolve_or_create_task( ) return task, False - profile = registry.resolve_assignment(db, owner_user_uuid) - if profile is None: + assignment = registry.resolve_assignment_context(db, owner_user_uuid) + if assignment is None: raise HTTPException( status_code=503, detail="No active agent profiles are configured on the server", ) + profile = assignment.profile + content_included = resolve_store_agent_content_for_acp(db, scope.user_id) task = crud.create_agent_task( db, @@ -177,6 +179,12 @@ def _resolve_or_create_task( agent_session_id=run.session_id, status="running", started_at=run.started_at, + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + study_arm_name=assignment.arm_name, + study_arm_is_baseline=assignment.is_baseline, + consent_content_storage=content_included, ) logging.info( f"[Agent/ingest] created task {task.task_id} for run {run.run_id} " diff --git a/src/backend/routers/agent/profiles.py b/src/backend/routers/agent/profiles.py index 3e17fac..1c04dc9 100644 --- a/src/backend/routers/agent/profiles.py +++ b/src/backend/routers/agent/profiles.py @@ -9,8 +9,8 @@ DELETE /profiles/{id} delete a profile (admin) GET /registry profiles in the shape the plugin consumes GET /assignments list A/B assignments (admin) - PUT /assignments/{user_id} pin a user to a profile (admin) - DELETE /assignments/{user_id} clear an assignment, re-rolling on next task + PUT /assignments/{user_id} add a study-specific assignment (admin) + DELETE /assignments/{user_id} clear a user's assignments (admin) Profiles are the unit of experimental control: a profile fixes the runtime, the provider, the model, the tool allowlist and the approval policy, and users are @@ -39,7 +39,13 @@ require_admin, ) from database import crud -from database.db_schemas import AgentProfile, AgentProfileAssignment # noqa: TC001 +from database.db_schemas import ( + AgentProfile, + AgentStudyAssignment, + Study, + StudyAgentProfile, + User, +) router = APIRouter() @@ -305,11 +311,9 @@ def update_agent_profile( ): """Update a profile. - Note this does *not* retroactively change tasks already running: the - experimental conditions (model, temperature, policy, tools) were snapshotted - onto each ``agent_task`` row at creation. The provider triple is the - exception and is read live, so rotating a credential or endpoint takes - effect on the next call. + A profile used by an active study cannot change. Agent tasks also copy the + effective runtime configuration at creation, so later changes never alter + completed task records. """ db = app.get_db_session() try: @@ -338,23 +342,23 @@ def update_agent_profile( raise HTTPException( status_code=409, detail="Agent profile name already exists" ) from exc + except crud.AgentProfileInActiveStudyError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail="Profiles attached to an active study cannot be changed", + ) from exc finally: db.close() -@router.delete("/profiles/{profile_id}", summary="Delete an agent profile") +@router.delete("/profiles/{profile_id}", summary="Retire an agent profile") def delete_agent_profile( profile_id: uuid.UUID, current_user: AuthenticatedUser = Depends(require_admin), app: App = Depends(App.get_instance), ): - """Delete a profile. - - Deleting cascades to its assignments but leaves historical ``agent_task`` - rows intact (they reference the profile by name, not id), so past telemetry - stays attributable. To retire an arm mid-study prefer setting - ``is_active=false``, which stops new draws without disturbing assigned users. - """ + """Retire a profile that is not part of an active study.""" db = app.get_db_session() try: deleted = crud.delete_agent_profile(db, profile_id) @@ -362,8 +366,14 @@ def delete_agent_profile( raise HTTPException(status_code=404, detail="Agent profile not found") return JsonResponseWithStatus( status_code=200, - content={"deleted": True, "profile_id": str(profile_id)}, + content={"retired": True, "profile_id": str(profile_id)}, ) + except crud.AgentProfileInActiveStudyError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail="Profiles attached to an active study cannot be retired", + ) from exc finally: db.close() @@ -394,13 +404,19 @@ def list_agent_registry( class AssignmentPayload(BaseModel): profile_id: uuid.UUID + study_id: Optional[uuid.UUID] = None -def _assignment_to_dict(assignment: AgentProfileAssignment) -> dict[str, Any]: +def _assignment_to_dict(assignment: AgentStudyAssignment) -> dict[str, Any]: + profile = assignment.profile return { + "assignment_id": str(assignment.assignment_id), + "study_id": str(assignment.study_id), "user_id": str(assignment.user_id), "profile_id": str(assignment.profile_id), - "profile_name": assignment.profile.name if assignment.profile else None, + "profile_name": profile.name if profile is not None else None, + "arm_name": assignment.arm_name, + "is_baseline": assignment.is_baseline, "source": assignment.source, "assigned_at": ( assignment.assigned_at.isoformat() @@ -417,7 +433,7 @@ def list_agent_assignments( ): db = app.get_db_session() try: - assignments = crud.list_agent_profile_assignments(db) + assignments = crud.list_agent_study_assignments(db) return JsonResponseWithStatus( status_code=200, content={"assignments": [_assignment_to_dict(a) for a in assignments]}, @@ -426,8 +442,44 @@ def list_agent_assignments( db.close() +@router.get("/assignment-options", summary="List users and studies for agent assignments") +def list_agent_assignment_options( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + users = db.query(User).order_by(User.name, User.email).all() + studies = db.query(Study).order_by(Study.is_active.desc(), Study.name).all() + profile_ids_by_study: dict[uuid.UUID, list[str]] = {} + for link in db.query(StudyAgentProfile).all(): + profile_ids_by_study.setdefault(link.study_id, []).append( + str(link.profile_id) + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "users": [ + {"user_id": str(user.user_id), "name": user.name, "email": user.email} + for user in users + ], + "studies": [ + { + "study_id": str(study.study_id), + "name": study.name, + "is_active": study.is_active, + "profile_ids": profile_ids_by_study.get(study.study_id, []), + } + for study in studies + ], + }, + ) + finally: + db.close() + + @router.put( - "/assignments/{user_id}", summary="Manually pin a user to an agent profile" + "/assignments/{user_id}", summary="Add an agent study assignment" ) def set_agent_assignment( user_id: uuid.UUID, @@ -437,30 +489,60 @@ def set_agent_assignment( ): db = app.get_db_session() try: - if crud.get_agent_profile_by_id(db, payload.profile_id) is None: + study_id = payload.study_id + if study_id is None: + active_study = crud.get_active_agent_study(db) + if active_study is None: + raise HTTPException( + status_code=409, + detail="No active study with agent arms; specify an active study first", + ) + study_id = active_study.study_id + arm = crud.get_study_agent_profile(db, study_id, payload.profile_id) + if arm is None: + raise HTTPException( + status_code=400, + detail="Profile is not an arm of the specified study", + ) + profile = crud.get_agent_profile_by_id(db, payload.profile_id) + if profile is None: raise HTTPException(status_code=404, detail="Agent profile not found") - assignment = crud.set_agent_profile_assignment( - db, user_id=user_id, profile_id=payload.profile_id, source="manual" + if crud.get_agent_study_assignment(db, study_id, user_id) is not None: + raise HTTPException( + status_code=409, + detail="Study assignment already exists and cannot be changed", + ) + assignment = crud.create_agent_study_assignment( + db, + study_id=study_id, + user_id=user_id, + profile_id=profile.profile_id, + arm_name=profile.name, + is_baseline=arm.is_baseline, + source="manual", ) + if assignment is None: + raise HTTPException( + status_code=409, + detail="Study assignment was created concurrently; reload assignments", + ) return JsonResponseWithStatus( - status_code=200, content={"assignment": _assignment_to_dict(assignment)} + status_code=201, content={"assignment": _assignment_to_dict(assignment)} ) finally: db.close() -@router.delete( - "/assignments/{user_id}", - summary="Clear a user's assignment (re-rolls on next task)", -) +@router.delete("/assignments/{user_id}", summary="Clear a user's agent assignments") def delete_agent_assignment( user_id: uuid.UUID, + study_id: Optional[uuid.UUID] = Query(default=None), current_user: AuthenticatedUser = Depends(require_admin), app: App = Depends(App.get_instance), ): db = app.get_db_session() try: - deleted = crud.delete_agent_profile_assignment(db, user_id) + deleted = crud.delete_agent_study_assignments(db, user_id, study_id) if not deleted: raise HTTPException(status_code=404, detail="Assignment not found") return JsonResponseWithStatus( diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py index 1413971..707edbd 100644 --- a/src/backend/routers/agents.py +++ b/src/backend/routers/agents.py @@ -125,12 +125,13 @@ def create_agent_task( status_code=401, detail="Session is not associated with a user" ) - profile = registry.resolve_assignment(db, session.user_id) - if profile is None: + assignment = registry.resolve_assignment_context(db, session.user_id) + if assignment is None: raise HTTPException( status_code=503, detail="No active agent profiles are configured on the server", ) + profile = assignment.profile # task_description is the user's own words — content, so honour the # consent gate even at creation time. @@ -149,6 +150,12 @@ def create_agent_task( owner_user_id=session.user_id, task_id=body.task_id, source="plugin", + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + study_arm_name=assignment.arm_name, + study_arm_is_baseline=assignment.is_baseline, + consent_content_storage=content_included, ) logging.info( f"[Agent/task] created task_id={task.task_id} profile={profile.name!r} " @@ -209,13 +216,9 @@ async def run_agent_inference( # Resolved server-side from the stored preference, never from the # request body — see backend.routers.agent.consent. content_included = resolve_store_agent_content(db, session_id) - # The profile is looked up only for the provider triple, which isn't - # snapshotted onto the task (rotating a base_url or key should take - # effect immediately, unlike the experimental conditions). - profile = crud.get_agent_profile(db, task.agent_profile) + profile = task.profile or crud.get_agent_profile(db, task.agent_profile) base_url = profile.base_url if profile else None api_key_ref = profile.api_key_ref if profile else None - # Snapshot values needed after the session closes. task_snapshot = { "agent_profile": task.agent_profile, "model": task.model, @@ -350,7 +353,11 @@ def upload_agent_telemetry( f"steps={len(child_spans)} tokens={total_prompt}/{total_completion}" ) - base_index = crud.count_agent_events_for_task(db, task_id) + base_index = ( + crud.reserve_agent_event_indexes(db, task_id, len(child_spans)) + if child_spans + else 0 + ) for offset, span in enumerate(child_spans): event_type = _SPAN_EVENT_TYPE.get(span.name, "observation") attrs = span.attributes or {} @@ -369,6 +376,8 @@ def upload_agent_telemetry( event_index=base_index + offset, event_type=event_type, source="proxy", + source_event_id=span.span_id, + ignore_duplicate_source=True, latency_ms=span.duration_ms if span.duration_ms > 0 else None, span_id=span.span_id, parent_span_id=parent_span_uuid, diff --git a/src/backend/routers/analytics/studies.py b/src/backend/routers/analytics/studies.py index 17a870b..348d35d 100644 --- a/src/backend/routers/analytics/studies.py +++ b/src/backend/routers/analytics/studies.py @@ -11,7 +11,7 @@ """ from typing import Optional, List, Dict, Any -from datetime import datetime +from datetime import datetime, timezone import uuid import random @@ -51,6 +51,13 @@ class UpdateStudyRequest(BaseModel): is_active: Optional[bool] = None +def _parse_study_datetime(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + @router.post("/create") def create_study( study_request: CreateStudyRequest, @@ -70,10 +77,10 @@ def create_study( try: # Validate datetime formats try: - starts_at = datetime.fromisoformat(study_request.starts_at.replace('Z', '+00:00')) + starts_at = _parse_study_datetime(study_request.starts_at) ends_at = None if study_request.ends_at: - ends_at = datetime.fromisoformat(study_request.ends_at.replace('Z', '+00:00')) + ends_at = _parse_study_datetime(study_request.ends_at) if ends_at <= starts_at: raise HTTPException(status_code=400, detail="End time must be after start time") except ValueError: @@ -121,7 +128,8 @@ def create_study( """ active_study = db_session.execute(text(active_study_query)).fetchone() - if active_study and starts_at <= datetime.now(): + now = datetime.now(timezone.utc) + if active_study and starts_at <= now: # Deactivate existing active study deactivate_query = """ UPDATE study SET is_active = false WHERE is_active = true @@ -130,7 +138,7 @@ def create_study( # Create new study study_id = uuid.uuid4() - is_active = starts_at <= datetime.now() and (ends_at is None or ends_at > datetime.now()) + is_active = starts_at <= now and (ends_at is None or ends_at > now) create_query = """ INSERT INTO study (study_id, name, description, created_by, starts_at, ends_at, is_active, default_config_id, created_at) @@ -146,7 +154,7 @@ def create_study( "ends_at": ends_at, "is_active": is_active, "default_config_id": study_request.default_config_id, - "created_at": datetime.now() + "created_at": now }) # If study is active, assign configurations to users @@ -183,6 +191,9 @@ def create_study( } ) + except HTTPException: + db_session.rollback() + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error creating study: {str(e)}") @@ -708,14 +719,13 @@ def evaluate_study_agents( Evaluate an A/B study's *agent* arms. The agent-side mirror of ``evaluate_study``: aggregates agent telemetry - (``agent_task`` + ``agent_event``) per attached agent profile, within the - study's time window, and reports uplift against the baseline arm. + (``agent_task`` + ``agent_event``) per snapshotted profile and reports + uplift against the baseline arm. - A task counts toward an arm when its ``agent_profile`` name matches a study - profile and it was created inside ``[starts_at, ends_at]``. Matching by name - rather than profile_id is deliberate — the name is what gets snapshotted - onto the task, so historical tasks stay attributable even if the profile row - is later edited or deleted. + A task counts only when its immutable ``study_id`` and ``profile_id`` + snapshots identify this study. Legacy tasks without those snapshots are + intentionally excluded rather than inferred from a mutable profile name or + a time window. Because both telemetry paths write into the same ``agent_event`` table, the same aggregation works whether an arm ran Goose, Codex, or the built-in @@ -747,30 +757,26 @@ def evaluate_study_agents( "ends_at": study.ends_at, } - # Per-arm task-level aggregates. LEFT JOIN so an arm with zero tasks - # still appears as a row with zeros, rather than vanishing from the - # comparison (an arm nobody used is itself a finding). + # Per-profile task aggregates use the immutable execution snapshot. + # Legacy tasks without a study/profile snapshot are excluded. task_query = """ SELECT - ap.profile_id, - ap.name AS profile_name, - ap.model, - ap.framework_version, - sap.is_baseline, + t.profile_id, + t.study_arm_name AS profile_name, + t.model, + t.framework_version, + t.study_arm_is_baseline AS is_baseline, COUNT(DISTINCT t.task_id) AS total_tasks, COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'done') AS completed_tasks, COUNT(DISTINCT COALESCE(t.owner_user_id, t.session_id)) AS total_participants, AVG(t.total_steps) AS avg_steps, SUM(t.input_tokens) AS total_input_tokens, SUM(t.output_tokens) AS total_output_tokens - FROM study_agent_profile sap - JOIN agent_profile ap ON ap.profile_id = sap.profile_id - LEFT JOIN agent_task t - ON t.agent_profile = ap.name - AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) - WHERE sap.study_id = :study_id - GROUP BY ap.profile_id, ap.name, ap.model, ap.framework_version, sap.is_baseline - ORDER BY ap.name + FROM agent_task t + WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL + GROUP BY t.profile_id, t.study_arm_name, t.model, t.framework_version, + t.study_arm_is_baseline + ORDER BY t.study_arm_name, t.profile_id """ task_rows = db_session.execute(text(task_query), window).fetchall() @@ -778,22 +784,18 @@ def evaluate_study_agents( # aggregates above aren't inflated by the event-row fan-out. event_query = """ SELECT - ap.name AS profile_name, + t.profile_id, COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call') AS avg_model_latency_ms, SUM(e.total_tokens) AS total_tokens - FROM study_agent_profile sap - JOIN agent_profile ap ON ap.profile_id = sap.profile_id - LEFT JOIN agent_task t - ON t.agent_profile = ap.name - AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) + FROM agent_task t LEFT JOIN agent_event e ON e.task_id = t.task_id - WHERE sap.study_id = :study_id - GROUP BY ap.name + WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL + GROUP BY t.profile_id """ event_rows = { - row.profile_name: row + str(row.profile_id): row for row in db_session.execute(text(event_query), window).fetchall() } @@ -803,22 +805,18 @@ def evaluate_study_agents( # edits but gets fewer accepted is worse, not better. edit_query = """ SELECT - ap.name AS profile_name, + t.profile_id, COUNT(ed.edit_id) AS total_edits, COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS TRUE) AS accepted_edits, COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS FALSE) AS rejected_edits, COUNT(ed.edit_id) FILTER (WHERE ed.was_modified IS TRUE) AS modified_edits - FROM study_agent_profile sap - JOIN agent_profile ap ON ap.profile_id = sap.profile_id - LEFT JOIN agent_task t - ON t.agent_profile = ap.name - AND t.created_at BETWEEN :starts_at AND COALESCE(:ends_at, NOW()) + FROM agent_task t LEFT JOIN agent_edit ed ON ed.task_id = t.task_id - WHERE sap.study_id = :study_id - GROUP BY ap.name + WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL + GROUP BY t.profile_id """ edit_rows = { - row.profile_name: row + str(row.profile_id): row for row in db_session.execute(text(edit_query), window).fetchall() } @@ -832,8 +830,8 @@ def _i(value) -> int: baseline_arm = None for row in task_rows: - events = event_rows.get(row.profile_name) - edits = edit_rows.get(row.profile_name) + events = event_rows.get(str(row.profile_id)) + edits = edit_rows.get(str(row.profile_id)) total_tasks = _i(row.total_tasks) total_edits = _i(edits.total_edits) if edits else 0 accepted_edits = _i(edits.accepted_edits) if edits else 0 diff --git a/src/database/crud.py b/src/database/crud.py index d8b9a2c..340023c 100644 --- a/src/database/crud.py +++ b/src/database/crud.py @@ -4,6 +4,7 @@ from typing import List, Optional, Tuple, Type, Union from sqlalchemy import func, text +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session import Queries as Queries @@ -13,6 +14,10 @@ from utils import hash_password, verify_password +class AgentProfileInActiveStudyError(Exception): + pass + + # User def create_user( db: Session, user: Union[Queries.CreateUser, Queries.CreateUserOauth] @@ -1293,39 +1298,48 @@ def update_agent_profile( max_context_tokens: Optional[int] = None, temperature: Optional[float] = None, ) -> Optional[db_schemas.AgentProfile]: - result = ( + profile = ( db.query(db_schemas.AgentProfile) .filter(db_schemas.AgentProfile.profile_id == profile_id) - .update( - { - "name": name, - "model": model, - "framework_version": framework_version, - "base_url": base_url, - "api_key_ref": api_key_ref, - "tools_json": tools_json, - "approval_policy": approval_policy, - "max_steps": max_steps, - "is_active": is_active, - "max_context_tokens": max_context_tokens, - "temperature": temperature, - } - ) + .with_for_update() + .first() ) + if profile is None: + return None + if is_agent_profile_in_active_study(db, profile_id): + raise AgentProfileInActiveStudyError + profile.name = name + profile.model = model + profile.framework_version = framework_version + profile.base_url = base_url + profile.api_key_ref = api_key_ref + profile.tools_json = tools_json + profile.approval_policy = approval_policy + profile.max_steps = max_steps + profile.is_active = is_active + profile.max_context_tokens = max_context_tokens + profile.temperature = temperature db.commit() - if result: - return get_agent_profile_by_id(db, profile_id) - return None + db.refresh(profile) + return profile def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: - result = ( + profile = ( db.query(db_schemas.AgentProfile) .filter(db_schemas.AgentProfile.profile_id == profile_id) - .delete() + .with_for_update() + .first() ) + if profile is None: + return False + if is_agent_profile_in_active_study(db, profile_id): + raise AgentProfileInActiveStudyError + if not profile.is_active: + return True + profile.is_active = False db.commit() - return result > 0 + return True # ── Agent profile assignments (A/B testing) ───────────────────────────────── @@ -1341,6 +1355,73 @@ def get_agent_profile_assignment( ) +def get_agent_study_assignment( + db: Session, study_id: uuid.UUID, user_id: uuid.UUID +) -> Optional[db_schemas.AgentStudyAssignment]: + return ( + db.query(db_schemas.AgentStudyAssignment) + .filter( + db_schemas.AgentStudyAssignment.study_id == study_id, + db_schemas.AgentStudyAssignment.user_id == user_id, + ) + .first() + ) + + +def create_agent_study_assignment( + db: Session, + *, + study_id: uuid.UUID, + user_id: uuid.UUID, + profile_id: uuid.UUID, + arm_name: str, + is_baseline: bool, + source: str, +) -> Optional[db_schemas.AgentStudyAssignment]: + assignment_id = uuid.uuid4() + inserted_assignment_id = db.execute( + pg_insert(db_schemas.AgentStudyAssignment) + .values( + assignment_id=assignment_id, + study_id=study_id, + user_id=user_id, + profile_id=profile_id, + arm_name=arm_name, + is_baseline=is_baseline, + source=source, + ) + .on_conflict_do_nothing(index_elements=["study_id", "user_id"]) + .returning(db_schemas.AgentStudyAssignment.assignment_id) + ).scalar_one_or_none() + if inserted_assignment_id is None: + return None + db.commit() + return db.get(db_schemas.AgentStudyAssignment, inserted_assignment_id) + + +def list_agent_study_assignments( + db: Session, +) -> List[db_schemas.AgentStudyAssignment]: + return ( + db.query(db_schemas.AgentStudyAssignment) + .order_by(db_schemas.AgentStudyAssignment.assigned_at.desc()) + .all() + ) + + +def delete_agent_study_assignments( + db: Session, user_id: uuid.UUID, study_id: Optional[uuid.UUID] = None +) -> int: + query = db.query(db_schemas.AgentStudyAssignment).filter( + db_schemas.AgentStudyAssignment.user_id == user_id + ) + if study_id is not None: + query = query.filter(db_schemas.AgentStudyAssignment.study_id == study_id) + deleted = query.delete() + db.commit() + return deleted + + def set_agent_profile_assignment( db: Session, user_id: uuid.UUID, @@ -1425,6 +1506,52 @@ def list_study_agent_profiles( ) +def get_study_agent_profile( + db: Session, study_id: uuid.UUID, profile_id: uuid.UUID +) -> Optional[db_schemas.StudyAgentProfile]: + return ( + db.query(db_schemas.StudyAgentProfile) + .filter( + db_schemas.StudyAgentProfile.study_id == study_id, + db_schemas.StudyAgentProfile.profile_id == profile_id, + ) + .first() + ) + + +def get_active_agent_study(db: Session) -> Optional[db_schemas.Study]: + """Return the active study when it has at least one active agent arm.""" + return ( + db.query(db_schemas.Study) + .join( + db_schemas.StudyAgentProfile, + db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, + ) + .join( + db_schemas.AgentProfile, + db_schemas.AgentProfile.profile_id + == db_schemas.StudyAgentProfile.profile_id, + ) + .filter(db_schemas.Study.is_active.is_(True)) + .filter(db_schemas.AgentProfile.is_active.is_(True)) + .first() + ) + + +def is_agent_profile_in_active_study(db: Session, profile_id: uuid.UUID) -> bool: + return ( + db.query(db_schemas.StudyAgentProfile) + .join( + db_schemas.Study, + db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, + ) + .filter(db_schemas.StudyAgentProfile.profile_id == profile_id) + .filter(db_schemas.Study.is_active.is_(True)) + .first() + is not None + ) + + def list_active_study_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: """Candidate arms drawn from the currently-active study's selected profiles. @@ -1452,6 +1579,28 @@ def list_active_study_agent_profiles(db: Session) -> List[db_schemas.AgentProfil ) +def list_active_study_agent_profile_links( + db: Session, +) -> List[db_schemas.StudyAgentProfile]: + """Return the arm links of the active agent study, if one exists.""" + return ( + db.query(db_schemas.StudyAgentProfile) + .join( + db_schemas.Study, + db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, + ) + .join( + db_schemas.AgentProfile, + db_schemas.AgentProfile.profile_id + == db_schemas.StudyAgentProfile.profile_id, + ) + .filter(db_schemas.Study.is_active.is_(True)) + .filter(db_schemas.AgentProfile.is_active.is_(True)) + .order_by(db_schemas.AgentProfile.name) + .all() + ) + + # ── Agent tasks ───────────────────────────────────────────────────────────── @@ -1474,6 +1623,12 @@ def create_agent_task( status: str = "pending", started_at: Optional[datetime] = None, policy_snapshot: Optional[dict] = None, + study_id: Optional[uuid.UUID] = None, + study_assignment_id: Optional[uuid.UUID] = None, + profile_id: Optional[uuid.UUID] = None, + study_arm_name: Optional[str] = None, + study_arm_is_baseline: Optional[bool] = None, + consent_content_storage: Optional[bool] = None, ) -> db_schemas.AgentTask: task = db_schemas.AgentTask( task_id=task_id or uuid.uuid4(), @@ -1490,6 +1645,12 @@ def create_agent_task( owner_project_id=owner_project_id, external_run_id=external_run_id, agent_session_id=agent_session_id, + study_id=study_id, + study_assignment_id=study_assignment_id, + profile_id=profile_id, + study_arm_name=study_arm_name, + study_arm_is_baseline=study_arm_is_baseline, + consent_content_storage=consent_content_storage, started_at=started_at, policy_snapshot=policy_snapshot, task_description=task_description, @@ -1575,7 +1736,7 @@ def update_agent_task_tools( ) -> None: db.query(db_schemas.AgentTask).filter( db_schemas.AgentTask.task_id == task_id - ).update({"tools_json": json.dumps(tools)}) + ).update({"observed_tools_json": json.dumps(tools)}) db.commit() @@ -1593,7 +1754,7 @@ def set_agent_task_framework_version( ) -> None: db.query(db_schemas.AgentTask).filter( db_schemas.AgentTask.task_id == task_id - ).update({"framework_version": framework_version}) + ).update({"observed_framework_version": framework_version}) db.commit() @@ -1632,7 +1793,7 @@ def get_last_agent_event_for_session( def count_agent_events_for_task(db: Session, task_id: uuid.UUID) -> int: - """Next free ``event_index`` for a task (events are 0-indexed and append-only).""" + """Return the number of persisted events for a task.""" return ( db.query(func.count(db_schemas.AgentEvent.event_id)) .filter(db_schemas.AgentEvent.task_id == task_id) @@ -1641,20 +1802,50 @@ def count_agent_events_for_task(db: Session, task_id: uuid.UUID) -> int: ) -def find_existing_agent_event_span_ids( - db: Session, span_ids: List[str] +def reserve_agent_event_indexes( + db: Session, task_id: uuid.UUID, event_count: int +) -> int: + """Atomically reserve ``event_count`` task-local event indexes.""" + if event_count < 1: + raise ValueError("event_count must be positive") + first_index = db.execute( + text( + """ + UPDATE public.agent_task + SET next_event_index = next_event_index + :event_count + WHERE task_id = :task_id + RETURNING next_event_index - :event_count + """ + ), + {"event_count": event_count, "task_id": task_id}, + ).scalar_one_or_none() + if first_index is None: + raise ValueError(f"Agent task {task_id} does not exist") + return int(first_index) + + +def find_existing_agent_event_source_ids( + db: Session, + *, + task_id: uuid.UUID, + source: str, + source_event_ids: List[str], ) -> List[str]: - """Which of ``span_ids`` have already been ingested for this task set. + """Return source event ids already persisted for this task and source. - Used to make self-report ingestion idempotent: a retried batch is detected - rather than duplicated. + This lookup is advisory only. The matching unique constraint is the + authoritative protection against concurrent retries. """ - if not span_ids: + if not source_event_ids: return [] return [ - row.span_id - for row in db.query(db_schemas.AgentEvent.span_id) - .filter(db_schemas.AgentEvent.span_id.in_(span_ids)) + row.source_event_id + for row in db.query(db_schemas.AgentEvent.source_event_id) + .filter( + db_schemas.AgentEvent.task_id == task_id, + db_schemas.AgentEvent.source == source, + db_schemas.AgentEvent.source_event_id.in_(source_event_ids), + ) .all() ] @@ -1666,6 +1857,7 @@ def append_agent_event( event_type: str, latency_ms: Optional[int] = None, source: Optional[str] = None, + source_event_id: Optional[str] = None, schema_version: Optional[str] = None, occurred_at: Optional[datetime] = None, # Shared span identifiers (model_call and tool_call) @@ -1713,8 +1905,9 @@ def append_agent_event( extra_json: Optional[str] = None, # content (only when store_agent_content resolves True) payload_json: Optional[str] = None, + ignore_duplicate_source: bool = False, commit: bool = True, -) -> db_schemas.AgentEvent: +) -> Optional[db_schemas.AgentEvent]: """Append one event row. ``commit=False`` lets a caller stage a whole batch and commit once (used by @@ -1727,6 +1920,7 @@ def append_agent_event( event_index=event_index, event_type=event_type, source=source, + source_event_id=source_event_id, schema_version=schema_version, latency_ms=latency_ms, occurred_at=occurred_at, @@ -1769,6 +1963,22 @@ def append_agent_event( extra_json=extra_json, payload_json=payload_json, ) + if ignore_duplicate_source and source is not None and source_event_id is not None: + values = { + column.name: getattr(event, column.name) + for column in db_schemas.AgentEvent.__table__.columns + } + inserted_event_id = db.execute( + pg_insert(db_schemas.AgentEvent) + .values(**values) + .on_conflict_do_nothing(constraint="uq_agent_event_source_identity") + .returning(db_schemas.AgentEvent.event_id) + ).scalar_one_or_none() + if inserted_event_id is None: + return None + if commit: + db.commit() + return db.get(db_schemas.AgentEvent, inserted_event_id) db.add(event) if commit: db.commit() diff --git a/src/database/db_schemas.py b/src/database/db_schemas.py index 36f2ba7..ed40d98 100644 --- a/src/database/db_schemas.py +++ b/src/database/db_schemas.py @@ -30,6 +30,7 @@ PrimaryKeyConstraint, String, Text, + UniqueConstraint, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import relationship @@ -816,6 +817,43 @@ class AgentProfileAssignment(Base): profile = relationship("AgentProfile") +class AgentStudyAssignment(Base): + """User-to-arm assignment within one specific study.""" + + __tablename__ = "agent_study_assignment" + __table_args__ = ( + Index("idx_agent_study_assignment_study_id", "study_id"), + Index("idx_agent_study_assignment_profile_id", "profile_id"), + UniqueConstraint("study_id", "user_id", name="uq_agent_study_assignment"), + {"schema": "public"}, + ) + + assignment_id = Column(UUID(as_uuid=True), primary_key=True) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="RESTRICT"), + nullable=False, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="RESTRICT"), + nullable=False, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), + nullable=False, + ) + arm_name = Column(String, nullable=False) + is_baseline = Column(Boolean, server_default="false", default=False, nullable=False) + source = Column(String, nullable=False, server_default="auto") + assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + study = relationship("Study") + user = relationship("User") + profile = relationship("AgentProfile") + + class AgentTask(Base): """One row per agent session a developer starts. @@ -832,6 +870,9 @@ class AgentTask(Base): Index("idx_agent_task_status", "status"), Index("idx_agent_task_session_id", "session_id"), Index("idx_agent_task_owner_user_id", "owner_user_id"), + Index("idx_agent_task_study_id", "study_id"), + Index("idx_agent_task_study_assignment_id", "study_assignment_id"), + Index("idx_agent_task_profile_id", "profile_id"), {"schema": "public"}, ) @@ -855,6 +896,19 @@ class AgentTask(Base): # matched back to their task without forcing the runtime to adopt our ids. external_run_id = Column(String, unique=True, nullable=True) agent_session_id = Column(String, nullable=True) + study_id = Column(UUID(as_uuid=True), ForeignKey("public.study.study_id"), nullable=True) + study_assignment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_study_assignment.assignment_id", ondelete="SET NULL"), + nullable=True, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id"), + nullable=True, + ) + study_arm_name = Column(String, nullable=True) + study_arm_is_baseline = Column(Boolean, nullable=True) agent_profile = Column(String, nullable=False) model = Column(String, nullable=False) # Snapshotted from the assigned profile at task-creation time, like `model`, so @@ -868,6 +922,10 @@ class AgentTask(Base): framework_version = Column(String, nullable=True) # Complete, versioned managed-runtime policy. Nullable for legacy tasks. policy_snapshot = Column(JSONB, nullable=True) + observed_framework_version = Column(String, nullable=True) + observed_tools_json = Column(Text, nullable=True) + consent_content_storage = Column(Boolean, nullable=True) + next_event_index = Column(Integer, nullable=False, server_default="0") status = Column(String, nullable=False) # pending | running | done | failed created_at = Column(DateTime, default=datetime.now) started_at = Column(DateTime(timezone=True), nullable=True) @@ -895,6 +953,10 @@ class AgentTask(Base): # Content — written only when store_agent_content resolves True task_description = Column(Text, nullable=True) + study = relationship("Study") + study_assignment = relationship("AgentStudyAssignment") + profile = relationship("AgentProfile") + class AgentEvent(Base): """One row per agent step, from either telemetry path. @@ -914,6 +976,10 @@ class AgentEvent(Base): Index("idx_agent_event_task_id", "task_id"), Index("idx_agent_event_event_type", "event_type"), Index("idx_agent_event_created_at", "created_at"), + UniqueConstraint("task_id", "event_index", name="uq_agent_event_task_event_index"), + UniqueConstraint( + "task_id", "source", "source_event_id", name="uq_agent_event_source_identity" + ), {"schema": "public"}, ) @@ -930,6 +996,7 @@ class AgentEvent(Base): # Which telemetry path wrote this row: "proxy" (client-side relay observing # a third-party agent) or "code4me2_agent" (runtime self-report). source = Column(String, nullable=True) + source_event_id = Column(String, nullable=True) # Envelope schema version reported by a self-reporting runtime, so old # batches stay interpretable after the envelope evolves. schema_version = Column(String, nullable=True) diff --git a/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py b/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py new file mode 100644 index 0000000..3509928 --- /dev/null +++ b/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py @@ -0,0 +1,328 @@ +"""add experiment execution integrity + +Adds study-scoped agent assignments and links new agent tasks to the exact +study arm, profile, and consent decision used at creation time. Task rows keep +their existing copied configuration as the immutable execution snapshot. Event +writes gain a task-local atomic index allocator and database-enforced +identity/order constraints. + +Legacy assignments and tasks are intentionally retained without inferred study +attribution. Existing events retain a NULL source_event_id because their source +identity cannot be reconstructed safely. + +Revision ID: f3c4d5e6f7a9 +Revises: f3c4d5e6a7b9 +Create Date: 2026-09-09 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +revision = "f3c4d5e6f7a9" +down_revision = "f3c4d5e6a7b9" +branch_labels = None +depends_on = None + +_SCHEMA = "public" + + +def upgrade() -> None: + op.create_table( + "agent_study_assignment", + sa.Column("assignment_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("arm_name", sa.String(), nullable=False), + sa.Column( + "is_baseline", sa.Boolean(), nullable=False, server_default=sa.text("false") + ), + sa.Column( + "source", sa.String(), nullable=False, server_default=sa.text("'auto'") + ), + sa.Column( + "assigned_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["study_id"], ["public.study.study_id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["user_id"], ["public.user.user_id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["profile_id"], + ["public.agent_profile.profile_id"], + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("assignment_id"), + sa.UniqueConstraint("study_id", "user_id", name="uq_agent_study_assignment"), + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_study_assignment_study_id", + "agent_study_assignment", + ["study_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_study_assignment_profile_id", + "agent_study_assignment", + ["profile_id"], + unique=False, + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("study_assignment_id", postgresql.UUID(as_uuid=True), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("study_arm_name", sa.String(), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("study_arm_is_baseline", sa.Boolean(), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("consent_content_storage", sa.Boolean(), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("observed_framework_version", sa.String(), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column("observed_tools_json", sa.Text(), nullable=True), + schema=_SCHEMA, + ) + op.add_column( + "agent_task", + sa.Column( + "next_event_index", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + schema=_SCHEMA, + ) + op.create_foreign_key( + "agent_task_study_id_fkey", + "agent_task", + "study", + ["study_id"], + ["study_id"], + source_schema=_SCHEMA, + referent_schema=_SCHEMA, + ) + op.create_foreign_key( + "agent_task_study_assignment_id_fkey", + "agent_task", + "agent_study_assignment", + ["study_assignment_id"], + ["assignment_id"], + source_schema=_SCHEMA, + referent_schema=_SCHEMA, + ondelete="SET NULL", + ) + op.create_foreign_key( + "agent_task_profile_id_fkey", + "agent_task", + "agent_profile", + ["profile_id"], + ["profile_id"], + source_schema=_SCHEMA, + referent_schema=_SCHEMA, + ) + op.create_index( + "idx_agent_task_study_id", + "agent_task", + ["study_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_task_study_assignment_id", + "agent_task", + ["study_assignment_id"], + unique=False, + schema=_SCHEMA, + ) + op.create_index( + "idx_agent_task_profile_id", + "agent_task", + ["profile_id"], + unique=False, + schema=_SCHEMA, + ) + op.execute( + """ + CREATE FUNCTION public.prevent_agent_task_snapshot_mutation() + RETURNS trigger AS $$ + BEGIN + IF OLD.profile_id IS NOT NULL AND ( + NEW.study_id IS DISTINCT FROM OLD.study_id OR + NEW.study_assignment_id IS DISTINCT FROM OLD.study_assignment_id OR + NEW.profile_id IS DISTINCT FROM OLD.profile_id OR + NEW.study_arm_name IS DISTINCT FROM OLD.study_arm_name OR + NEW.study_arm_is_baseline IS DISTINCT FROM OLD.study_arm_is_baseline OR + NEW.consent_content_storage IS DISTINCT FROM OLD.consent_content_storage OR + NEW.agent_profile IS DISTINCT FROM OLD.agent_profile OR + NEW.model IS DISTINCT FROM OLD.model OR + NEW.temperature IS DISTINCT FROM OLD.temperature OR + NEW.approval_policy IS DISTINCT FROM OLD.approval_policy OR + NEW.tools_json IS DISTINCT FROM OLD.tools_json OR + NEW.framework_version IS DISTINCT FROM OLD.framework_version OR + NEW.policy_snapshot IS DISTINCT FROM OLD.policy_snapshot + ) THEN + RAISE EXCEPTION 'Agent task execution snapshots are immutable'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER agent_task_snapshot_immutable + BEFORE UPDATE ON public.agent_task + FOR EACH ROW + EXECUTE FUNCTION public.prevent_agent_task_snapshot_mutation(); + """ + ) + + op.execute( + """ + UPDATE public.agent_task task + SET next_event_index = COALESCE( + ( + SELECT MAX(event.event_index) + 1 + FROM public.agent_event event + WHERE event.task_id = task.task_id + ), + 0 + ) + """ + ) + + op.add_column( + "agent_event", + sa.Column("source_event_id", sa.String(), nullable=True), + schema=_SCHEMA, + ) + + # Historical duplicate positions have no defensible automatic repair. Stop + # rather than silently selecting an arbitrary event as the canonical one. + op.execute( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM public.agent_event + GROUP BY task_id, event_index + HAVING COUNT(*) > 1 + ) THEN + RAISE EXCEPTION + 'Cannot add agent event ordering constraint: duplicate task/event indexes exist'; + END IF; + END $$; + """ + ) + op.create_unique_constraint( + "uq_agent_event_task_event_index", + "agent_event", + ["task_id", "event_index"], + schema=_SCHEMA, + ) + op.create_unique_constraint( + "uq_agent_event_source_identity", + "agent_event", + ["task_id", "source", "source_event_id"], + schema=_SCHEMA, + ) + + +def downgrade() -> None: + op.execute("DROP TRIGGER agent_task_snapshot_immutable ON public.agent_task") + op.execute("DROP FUNCTION public.prevent_agent_task_snapshot_mutation()") + op.drop_constraint( + "uq_agent_event_source_identity", + "agent_event", + type_="unique", + schema=_SCHEMA, + ) + op.drop_constraint( + "uq_agent_event_task_event_index", + "agent_event", + type_="unique", + schema=_SCHEMA, + ) + op.drop_column("agent_event", "source_event_id", schema=_SCHEMA) + + op.drop_index( + "idx_agent_task_study_assignment_id", table_name="agent_task", schema=_SCHEMA + ) + op.drop_index("idx_agent_task_profile_id", table_name="agent_task", schema=_SCHEMA) + op.drop_index("idx_agent_task_study_id", table_name="agent_task", schema=_SCHEMA) + op.drop_constraint( + "agent_task_profile_id_fkey", + "agent_task", + type_="foreignkey", + schema=_SCHEMA, + ) + op.drop_constraint( + "agent_task_study_assignment_id_fkey", + "agent_task", + type_="foreignkey", + schema=_SCHEMA, + ) + op.drop_constraint( + "agent_task_study_id_fkey", + "agent_task", + type_="foreignkey", + schema=_SCHEMA, + ) + for column_name in ( + "next_event_index", + "consent_content_storage", + "study_arm_is_baseline", + "study_arm_name", + "profile_id", + "study_assignment_id", + "study_id", + "observed_tools_json", + "observed_framework_version", + ): + op.drop_column("agent_task", column_name, schema=_SCHEMA) + + op.drop_index( + "idx_agent_study_assignment_profile_id", + table_name="agent_study_assignment", + schema=_SCHEMA, + ) + op.drop_index( + "idx_agent_study_assignment_study_id", + table_name="agent_study_assignment", + schema=_SCHEMA, + ) + op.drop_table("agent_study_assignment", schema=_SCHEMA) diff --git a/src/website/src/components/analytics/StudyManagement.css b/src/website/src/components/analytics/StudyManagement.css index 4883c2c..ee498f6 100644 --- a/src/website/src/components/analytics/StudyManagement.css +++ b/src/website/src/components/analytics/StudyManagement.css @@ -457,6 +457,69 @@ font-family: inherit; } +.study-option-group { + margin: 0; + padding: 12px; + border: 1px solid var(--border-color); + border-radius: 6px; +} + +.study-option-group legend { + padding: 0 4px; + color: var(--text-color); + font-size: 14px; + font-weight: 500; +} + +.study-options-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); + gap: 8px; +} + +.study-option { + display: flex; + align-items: center; + gap: 8px; + padding: 8px; + border: 1px solid var(--border-color); + border-radius: 4px; + color: var(--text-color); + cursor: pointer; +} + +.study-option input { + width: auto; +} + +.form-help { + margin: 6px 0 0; + color: var(--text-secondary); + font-size: 12px; + line-height: 1.4; +} + +.baseline-select { + display: flex; + flex-direction: column; + gap: 6px; + margin-top: 12px; + color: var(--text-color); + font-size: 13px; + font-weight: 500; +} + +.baseline-select select, +.form-group select { + width: 100%; + padding: 8px 12px; + border: 1px solid var(--border-color); + border-radius: 4px; + background-color: var(--secondary-color); + color: var(--text-color); + font-size: 14px; +} + .form-actions { display: flex; gap: 12px; diff --git a/src/website/src/components/analytics/StudyManagement.js b/src/website/src/components/analytics/StudyManagement.js index e193ba2..6b86aa8 100644 --- a/src/website/src/components/analytics/StudyManagement.js +++ b/src/website/src/components/analytics/StudyManagement.js @@ -1,5 +1,5 @@ import React, { useState, useEffect } from 'react'; -import { getStudies, getStudyEvaluation, getStudyDetails, activateStudy, deactivateStudy, createStudy, getStudyAgentEvaluation } from '../../utils/api'; +import { getStudies, getStudyEvaluation, getStudyDetails, activateStudy, deactivateStudy, createStudy, getStudyAgentEvaluation, getAgentProfiles, listConfigs } from '../../utils/api'; import AgentResults from './AgentResults'; import './StudyManagement.css'; @@ -411,15 +411,86 @@ const StudyManagement = ({ user }) => { // Simple Create Study Modal Component const CreateStudyModal = ({ onClose, onStudyCreated }) => { + const [availableConfigs, setAvailableConfigs] = useState([]); + const [availableProfiles, setAvailableProfiles] = useState([]); + const [isLoadingOptions, setIsLoadingOptions] = useState(true); const [formData, setFormData] = useState({ name: '', description: '', starts_at: new Date().toISOString().slice(0, 16), - config_ids: [1, 2], // Default configs - default_config_id: 1 + config_ids: [], + default_config_id: '', + agent_profile_ids: [], + baseline_agent_profile_id: null, }); const [isSubmitting, setIsSubmitting] = useState(false); + useEffect(() => { + let isCancelled = false; + + const loadStudyOptions = async () => { + const [configResponse, profileResponse] = await Promise.all([ + listConfigs(), + getAgentProfiles(), + ]); + + if (isCancelled) { + return; + } + + const configs = configResponse.ok ? configResponse.data : []; + const profiles = profileResponse.ok + ? profileResponse.data.filter((profile) => profile.is_active) + : []; + const configIds = configs.map((config) => config.config_id); + const initialConfigIds = configIds.slice(0, 2); + + setAvailableConfigs(configs); + setAvailableProfiles(profiles); + setFormData((current) => ({ + ...current, + config_ids: initialConfigIds, + default_config_id: initialConfigIds[0] || '', + })); + setIsLoadingOptions(false); + }; + + loadStudyOptions().catch((error) => { + console.error("Study options error:", error); + if (!isCancelled) { + setIsLoadingOptions(false); + } + }); + + return () => { + isCancelled = true; + }; + }, []); + + const toggleConfig = (configId) => { + setFormData((current) => { + const configIds = current.config_ids.includes(configId) + ? current.config_ids.filter((id) => id !== configId) + : [...current.config_ids, configId]; + const defaultConfigId = configIds.includes(current.default_config_id) + ? current.default_config_id + : configIds[0] || ''; + return { ...current, config_ids: configIds, default_config_id: defaultConfigId }; + }); + }; + + const toggleAgentProfile = (profileId) => { + setFormData((current) => { + const agentProfileIds = current.agent_profile_ids.includes(profileId) + ? current.agent_profile_ids.filter((id) => id !== profileId) + : [...current.agent_profile_ids, profileId]; + const baselineAgentProfileId = agentProfileIds.includes(current.baseline_agent_profile_id) + ? current.baseline_agent_profile_id + : agentProfileIds[0] || null; + return { ...current, agent_profile_ids: agentProfileIds, baseline_agent_profile_id: baselineAgentProfileId }; + }); + }; + const handleSubmit = async (e) => { e.preventDefault(); setIsSubmitting(true); @@ -481,34 +552,92 @@ const CreateStudyModal = ({ onClose, onStudyCreated }) => {
- - { - const ids = e.target.value.split(',').map(id => parseInt(id.trim())).filter(id => !isNaN(id)); - setFormData({ ...formData, config_ids: ids }); - }} - placeholder="1, 2, 3" - required - /> +
+ Completion config arms * + {isLoadingOptions ? ( +

Loading available configs...

+ ) : availableConfigs.length === 0 ? ( +

Create a completion config before starting a study.

+ ) : ( +
+ {availableConfigs.map((config) => ( + + ))} +
+ )} +
- setFormData({ ...formData, default_config_id: parseInt(e.target.value) })} required - /> + disabled={!formData.config_ids.length} + > + {formData.config_ids.map((configId) => ( + + ))} + +
+ +
+
+ Agent profile arms (optional) +

Select active profiles to randomize agent users between arms.

+ {isLoadingOptions ? ( +

Loading active agent profiles...

+ ) : availableProfiles.length === 0 ? ( +

Create and activate an agent profile to add an agent arm.

+ ) : ( +
+ {availableProfiles.map((profile) => ( + + ))} +
+ )} + {formData.agent_profile_ids.length > 0 && ( + + )} +
-
diff --git a/src/website/src/pages/AgentAssignments.js b/src/website/src/pages/AgentAssignments.js index f2b3715..cf49213 100644 --- a/src/website/src/pages/AgentAssignments.js +++ b/src/website/src/pages/AgentAssignments.js @@ -2,6 +2,7 @@ import React, { useEffect, useMemo, useState } from "react"; import { deleteAgentAssignment, getAgentAssignments, + getAgentAssignmentOptions, getAgentProfiles, setAgentAssignment, } from "../utils/api"; @@ -18,8 +19,11 @@ const shortId = (id) => (id ? `${String(id).slice(0, 8)}…` : "—"); const AgentAssignments = () => { const [assignments, setAssignments] = useState([]); const [profiles, setProfiles] = useState([]); + const [users, setUsers] = useState([]); + const [studies, setStudies] = useState([]); const [pinUserId, setPinUserId] = useState(""); const [pinProfileId, setPinProfileId] = useState(""); + const [pinStudyId, setPinStudyId] = useState(""); const [isLoading, setIsLoading] = useState(true); const [isSaving, setIsSaving] = useState(false); const [error, setError] = useState(""); @@ -33,6 +37,37 @@ const AgentAssignments = () => { return map; }, [profiles]); + const userById = useMemo( + () => Object.fromEntries(users.map((user) => [user.user_id, user])), + [users], + ); + + const studyById = useMemo( + () => Object.fromEntries(studies.map((study) => [study.study_id, study])), + [studies], + ); + + const selectedStudy = useMemo( + () => + studyById[pinStudyId] || studies.find((study) => study.is_active) || null, + [pinStudyId, studies, studyById], + ); + + const eligibleProfiles = useMemo(() => { + if (!selectedStudy) return profiles; + const profileIds = new Set(selectedStudy.profile_ids || []); + return profiles.filter((profile) => profileIds.has(profile.profile_id || profile.id)); + }, [profiles, selectedStudy]); + + useEffect(() => { + if ( + pinProfileId && + !eligibleProfiles.some((profile) => (profile.profile_id || profile.id) === pinProfileId) + ) { + setPinProfileId(""); + } + }, [eligibleProfiles, pinProfileId]); + // Which runtime each arm targets. Worth surfacing here because arms in the // same study can now run entirely different agents (Goose vs Codex vs the // built-in runtime), and that is usually the variable under test. @@ -44,24 +79,39 @@ const AgentAssignments = () => { return map; }, [profiles]); - // Count of users assigned to each profile, split by how they got there. + // Treatment distribution is meaningful only within a study. const distribution = useMemo(() => { const counts = {}; assignments.forEach((a) => { - const key = a.profile_name || profileNameById[a.profile_id] || "unknown"; - if (!counts[key]) counts[key] = { auto: 0, manual: 0, total: 0 }; + const profileName = + a.profile_name || profileNameById[a.profile_id] || "unknown"; + const key = `${a.study_id}:${profileName}`; + if (!counts[key]) { + counts[key] = { + auto: 0, + manual: 0, + total: 0, + profileName, + studyId: a.study_id, + }; + } counts[key][a.source === "manual" ? "manual" : "auto"] += 1; counts[key].total += 1; }); - return Object.entries(counts).sort((a, b) => a[0].localeCompare(b[0])); + return Object.values(counts).sort((a, b) => + `${a.studyId}:${a.profileName}`.localeCompare( + `${b.studyId}:${b.profileName}`, + ), + ); }, [assignments, profileNameById]); const loadData = async () => { setIsLoading(true); setError(""); - const [assignmentsRes, profilesRes] = await Promise.all([ + const [assignmentsRes, profilesRes, optionsRes] = await Promise.all([ getAgentAssignments(), getAgentProfiles(), + getAgentAssignmentOptions(), ]); if (assignmentsRes.ok) { setAssignments(assignmentsRes.data || []); @@ -73,6 +123,12 @@ const AgentAssignments = () => { Array.isArray(profilesRes.data) ? profilesRes.data : [], ); } + if (optionsRes.ok) { + setUsers(optionsRes.data.users || []); + setStudies(optionsRes.data.studies || []); + } else { + setError(optionsRes.error); + } setIsLoading(false); }; @@ -82,8 +138,8 @@ const AgentAssignments = () => { const handlePin = async (event) => { event.preventDefault(); - if (!pinUserId.trim()) { - setError("User ID is required."); + if (!pinUserId) { + setError("Pick a user."); return; } if (!pinProfileId) { @@ -94,11 +150,16 @@ const AgentAssignments = () => { setIsSaving(true); setError(""); setNotice(""); - const response = await setAgentAssignment(pinUserId.trim(), pinProfileId); + const response = await setAgentAssignment( + pinUserId, + pinProfileId, + pinStudyId || undefined, + ); if (response.ok) { - setNotice("User pinned to profile."); + setNotice("Study assignment created."); setPinUserId(""); setPinProfileId(""); + setPinStudyId(""); await loadData(); } else { setError(response.error); @@ -106,31 +167,16 @@ const AgentAssignments = () => { setIsSaving(false); }; - const handleReassign = async (userId, profileId) => { - if (!profileId) return; - setIsSaving(true); - setError(""); - setNotice(""); - const response = await setAgentAssignment(userId, profileId); - if (response.ok) { - setNotice("Assignment updated (manual)."); - await loadData(); - } else { - setError(response.error); - } - setIsSaving(false); - }; - - const handleClear = async (userId) => { + const handleClear = async (userId, studyId) => { const confirmed = window.confirm( - "Clear this assignment? The user will be re-bucketed at random on their next agent task.", + "Clear this assignment? The user will be re-drawn on their next agent task.", ); if (!confirmed) return; setIsSaving(true); setError(""); setNotice(""); - const response = await deleteAgentAssignment(userId); + const response = await deleteAgentAssignment(userId, studyId); if (response.ok) { setNotice("Assignment cleared."); await loadData(); @@ -146,10 +192,9 @@ const AgentAssignments = () => {

Agent Assignments

- Which A/B variant each user is pinned to. Assignments are normally - drawn at random on a user's first agent task; pin a user manually - only for test accounts or debugging — manual pins are excluded from - A/B analysis. + A/B assignments, grouped by study. New assignments are + normally drawn on a user's first agent task; add a manual row only + for test accounts or debugging.

@@ -233,6 +311,7 @@ const AgentAssignments = () => { User + Study Profile Source Assigned at @@ -241,8 +320,15 @@ const AgentAssignments = () => { {assignments.map((a) => ( - - {shortId(a.user_id)} + + + {userById[a.user_id] + ? `${userById[a.user_id].name} (${userById[a.user_id].email})` + : shortId(a.user_id)} + + + {studyById[a.study_id]?.name || shortId(a.study_id)} + {a.profile_name || profileNameById[a.profile_id] || @@ -255,34 +341,13 @@ const AgentAssignments = () => { {formatDate(a.assigned_at)} -
- - -
+ ))} diff --git a/src/website/src/pages/AgentProfiles.js b/src/website/src/pages/AgentProfiles.js index 3b1db0f..9a4fcaf 100644 --- a/src/website/src/pages/AgentProfiles.js +++ b/src/website/src/pages/AgentProfiles.js @@ -279,10 +279,9 @@ const AgentProfiles = () => { const handleDelete = async (profile) => { const profileId = getProfileId(profile); const confirmed = window.confirm( - `Delete agent profile "${profile.name}"? Historical agent tasks reference ` + - `profiles by name, so past telemetry stays attributable — but any A/B ` + - `assignments pointing at this profile are removed. To retire an arm ` + - `mid-study, uncheck "Active" instead.`, + `Retire agent profile "${profile.name}"? It will no longer be assigned ` + + `to new participants, while its immutable version history and existing ` + + `study assignments remain available for analysis.`, ); if (!confirmed) return; @@ -292,7 +291,7 @@ const AgentProfiles = () => { const response = await deleteAgentProfile(profileId); if (response.ok) { - setNotice("Agent profile deleted."); + setNotice("Agent profile retired."); if (editingProfileId === profileId) resetForm(); await loadProfiles(); } else { diff --git a/src/website/src/utils/api.js b/src/website/src/utils/api.js index dc87f63..3b82ff2 100644 --- a/src/website/src/utils/api.js +++ b/src/website/src/utils/api.js @@ -347,7 +347,7 @@ export const resendVerificationEmail = async () => { } else { return { ok: true, - message: responseBody["message"] || "Verification email sent successfully", + message: responseBody["message"] || "Verification email request queued", }; } } catch (error) { @@ -1331,21 +1331,33 @@ export const getAgentAssignments = async () => { : result; }; -// Pin a user to a specific arm. Recorded as source="manual", which excludes -// the row from re-rolls and from A/B analysis. -export const setAgentAssignment = async (userId, profileId) => +export const getAgentAssignmentOptions = async () => { + const result = await agentRequest("/assignment-options", { + label: "load assignment options", + }); + return result.ok ? { ok: true, data: result.data } : result; +}; + +// Add a manual assignment to a study arm. +export const setAgentAssignment = async (userId, profileId, studyId) => agentRequest(`/assignments/${userId}`, { method: "PUT", - body: { profile_id: profileId }, + body: { + profile_id: profileId, + ...(studyId ? { study_id: studyId } : {}), + }, label: "set agent assignment", }); // Clear an assignment so the user is re-drawn on their next agent task. -export const deleteAgentAssignment = async (userId) => - agentRequest(`/assignments/${userId}`, { - method: "DELETE", - label: "clear agent assignment", - }); +export const deleteAgentAssignment = async (userId, studyId) => + agentRequest( + `/assignments/${userId}${studyId ? `?study_id=${encodeURIComponent(studyId)}` : ""}`, + { + method: "DELETE", + label: "clear agent assignment", + }, + ); // Per-arm comparison for a study's agent arms. export const getStudyAgentEvaluation = async (studyId) => { From 31847173fc81834a9fa8fa72d167898bfad24a23 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Sun, 13 Sep 2026 19:03:17 +0200 Subject: [PATCH 016/115] feat: Add approval prompts for Code4Me edits --- src/code4me2_agent/acp_runtime.py | 112 +++++++++--- src/code4me2_agent/acp_updates.py | 13 +- src/code4me2_agent/adapters.py | 147 ++++++++++++--- src/code4me2_agent/events.py | 12 ++ tests/code4me2_agent/test_acp_runtime.py | 171 +++++++++++++++++- .../code4me2_agent/test_policy_enforcement.py | 79 +++++--- 6 files changed, 460 insertions(+), 74 deletions(-) diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py index efd09b8..cd8d561 100644 --- a/src/code4me2_agent/acp_runtime.py +++ b/src/code4me2_agent/acp_runtime.py @@ -14,6 +14,7 @@ from code4me2_agent.async_bridge import EventLoopAsyncRunner from code4me2_agent.command_tools import available_commands, build_acp_command_backend from code4me2_agent.echo import EchoAgentCore +from code4me2_agent.events import ApprovalDecision from code4me2_agent.file_tools import build_acp_file_system_backend from code4me2_agent.mcp_tools import StdioMcpToolBroker, serialize_mcp_servers from code4me2_agent.runtime_auth import ( @@ -150,23 +151,23 @@ def __init__( self._updates = updates self._telemetry = telemetry self._async_runner = async_runner + self._session_approved_kinds: set[str] = set() + self._tool_content: dict[str, list[Any] | None] = {} def tool_call(self, event: ToolCallEvent) -> None: + content = self._tool_call_content(event) if event.phase == "started": + self._tool_content[event.tool_call_id] = content update = self._updates.start_tool_call( tool_call_id=event.tool_call_id, title=event.title, kind=event.kind, status=event.status, path=event.path, + content=content, raw_input=event.raw_input, ) else: - content = ( - [self._updates.text_tool_content(event.content_text)] - if event.content_text - else None - ) update = self._updates.update_tool_call( tool_call_id=event.tool_call_id, title=event.title, @@ -176,6 +177,21 @@ def tool_call(self, event: ToolCallEvent) -> None: raw_output=event.raw_output, ) self._send_update(event=event, update=update) + if event.phase != "started": + self._tool_content.pop(event.tool_call_id, None) + + def _tool_call_content(self, event: ToolCallEvent) -> list[Any] | None: + if event.path and event.diff_new_text is not None: + return [ + self._updates.diff_tool_content( + event.path, + old_text=event.diff_old_text, + new_text=event.diff_new_text, + ) + ] + if event.content_text: + return [self._updates.text_tool_content(event.content_text)] + return None def thought(self, event: object) -> None: phase = getattr(event, "phase", None) @@ -212,24 +228,30 @@ def thought(self, event: object) -> None: }, ) - def request_approval(self, tool_call: object, arguments: dict[str, Any]) -> bool: + def request_approval( + self, tool_call: object, arguments: dict[str, Any] + ) -> ApprovalDecision: """Synchronously bridge a worker-thread tool decision to ACP/JetBrains.""" name = str(getattr(tool_call, "name", "tool")) tool_call_id = str(getattr(tool_call, "tool_call_id", "")) metadata = { - "run_command": ("Execute command", "execute"), - "create_file": ("Create file", "edit"), - "write_file": ("Write file", "edit"), - "replace_text": ("Replace text", "edit"), - "read_file": ("Read file", "read"), - "list_files": ("List files", "search"), - "search_files": ("Search files", "search"), - }.get(name, (f"Call {name}", "other")) + "run_command": "execute", + "create_file": "edit", + "write_file": "edit", + "replace_text": "edit", + }.get(name, "other") + if metadata in self._session_approved_kinds: + return ApprovalDecision("accepted", "session") + summary = _approval_summary(name, arguments) permission = self._updates.permission_request( session_id=self._session_id, tool_call_id=tool_call_id, - title=metadata[0], - kind=metadata[1], + title=summary, + kind=metadata, + summary=summary, + raw_input=_approval_raw_input(name, arguments), + session_option_name=_session_option_name(metadata), + content=self._tool_content.get(tool_call_id), ) try: response = self._async_runner.run( @@ -241,19 +263,25 @@ def request_approval(self, tool_call: object, arguments: dict[str, Any]) -> bool ) except Exception: # noqa: BLE001 logger.exception("ACP permission request failed for tool %s", name) - return False + return ApprovalDecision("unavailable") outcome = getattr(response, "outcome", None) if isinstance(response, dict): outcome = response.get("outcome") if isinstance(outcome, dict): - return ( - outcome.get("outcome") == "selected" - and outcome.get("optionId", outcome.get("option_id")) == "allow_once" - ) - return ( - getattr(outcome, "outcome", None) == "selected" - and getattr(outcome, "option_id", None) == "allow_once" - ) + selected = outcome.get("outcome") == "selected" + option_id = outcome.get("optionId", outcome.get("option_id")) + else: + selected = getattr(outcome, "outcome", None) == "selected" + option_id = getattr(outcome, "option_id", None) + if not selected: + return ApprovalDecision("cancelled") + scope = { + "allow_once": "once", + "allow_session": "session", + }.get(option_id) + if scope == "session": + self._session_approved_kinds.add(metadata) + return ApprovalDecision("accepted", scope) if scope else ApprovalDecision("rejected") def _send_update(self, *, event: ToolCallEvent, update: object) -> None: try: @@ -298,6 +326,40 @@ def _record_acp_runtime_event( ) +def _approval_raw_input(name: str, arguments: dict[str, Any]) -> dict[str, Any] | None: + if name == "run_command": + argv = arguments.get("argv") + return { + "argv": [str(arg) for arg in argv] if isinstance(argv, (list, tuple)) else [], + "cwd": str(arguments.get("cwd", ".")), + } + if name in {"create_file", "write_file", "replace_text"}: + return {"path": str(arguments.get("path", ""))} + return None + + +def _approval_summary(name: str, arguments: dict[str, Any]) -> str: + if name == "run_command": + argv = arguments.get("argv") + command = " ".join(str(arg) for arg in argv) if isinstance(argv, (list, tuple)) else "command" + return f"Run command: {command}" + path = str(arguments.get("path", "")).strip() + labels = { + "create_file": "Create file", + "write_file": "Write file", + "replace_text": "Edit file", + } + return f"{labels.get(name, f'Run tool: {name}')}{f': {path}' if path else ''}" + + +def _session_option_name(kind: str) -> str: + return { + "edit": "Allow edits for session", + "execute": "Allow commands for session", + "other": "Allow MCP tools for session", + }[kind] + + def _normalize_capabilities(value: object | None) -> object | None: if value is None or isinstance(value, (str, int, float, bool)): return value diff --git a/src/code4me2_agent/acp_updates.py b/src/code4me2_agent/acp_updates.py index 8488e79..61efc70 100644 --- a/src/code4me2_agent/acp_updates.py +++ b/src/code4me2_agent/acp_updates.py @@ -186,6 +186,10 @@ def permission_request( tool_call_id: str, title: str, kind: str, + summary: str, + raw_input: Any | None = None, + session_option_name: str = "Allow for session", + content: Sequence[Any] | None = None, ) -> Any: return self._schema.RequestPermissionRequest( session_id=session_id, @@ -194,6 +198,8 @@ def permission_request( title=title, kind=kind, status="pending", + content=content or [self.text_tool_content(summary)], + raw_input=raw_input, ), options=[ self._schema.PermissionOption( @@ -201,9 +207,14 @@ def permission_request( name="Allow once", kind="allow_once", ), + self._schema.PermissionOption( + option_id="allow_session", + name=session_option_name, + kind="allow_always", + ), self._schema.PermissionOption( option_id="reject_once", - name="Reject once", + name="Reject", kind="reject_once", ), ], diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py index 52fd6f4..59b19ca 100644 --- a/src/code4me2_agent/adapters.py +++ b/src/code4me2_agent/adapters.py @@ -282,29 +282,44 @@ def execute( failure_reason="invalid_approval_policy", ) if self._approval_policy == "suggestion_only" and ( - name in {"create_file", "write_file", "replace_text", "run_command"} - or name.startswith("mcp__") + _requires_manual_approval(name) ): raise ToolRegistryError( f"Tool execution is disabled by suggestion-only policy: {name}", failure_reason="approval_policy_denied", ) arguments = dict(tool_call.arguments) - if self._approval_policy == "per_step": + edit_preview = self._edit_preview( + tool_call, arguments, run_id=run_id, request_id=request_id + ) + self._emit_tool_start( + tool_call, + arguments, + edit_preview=edit_preview, + run_id=run_id, + request_id=request_id, + ) + if self._approval_policy == "per_step" and _requires_manual_approval(name): request_approval = getattr(self._event_sink, "request_approval", None) - approved = bool( + decision = ( request_approval(tool_call, arguments) if callable(request_approval) - else False + else None ) - if not approved: + if not getattr(decision, "accepted", bool(decision)): + outcome = getattr(decision, "decision", "unavailable") + self._emit_tool_failed( + tool_call, + arguments, + edit_preview=edit_preview, + run_id=run_id, + request_id=request_id, + message=f"Not run: approval {outcome}.", + ) raise ToolRegistryError( - f"The user denied approval for tool: {name}", - failure_reason="approval_policy_denied", + f"Tool approval {outcome} for: {name}", + failure_reason=f"approval_{outcome}", ) - self._emit_tool_start( - tool_call, arguments, run_id=run_id, request_id=request_id - ) try: if name == "read_file": result = self._file_tools.read_file( @@ -372,15 +387,23 @@ def execute( ) except Exception: self._emit_tool_failed( - tool_call, arguments, run_id=run_id, request_id=request_id + tool_call, + arguments, + edit_preview=edit_preview, + run_id=run_id, + request_id=request_id, ) raise tool_output = asdict(result) if is_dataclass(result) else dict(result) self._emit_tool_completed( - tool_call, tool_output, run_id=run_id, request_id=request_id + tool_call, + arguments, + tool_output, + edit_preview=edit_preview, + run_id=run_id, + request_id=request_id, ) return tool_output - def definitions(self) -> list[dict[str, Any]]: definitions = _tool_definitions() if self._mcp_tools is not None: @@ -418,6 +441,7 @@ def _emit_tool_start( tool_call: ToolCall, arguments: dict[str, Any], *, + edit_preview: dict[str, str | None] | None = None, run_id: str, request_id: str, ) -> None: @@ -435,6 +459,8 @@ def _emit_tool_start( kind=metadata["kind"], status="pending", path=metadata.get("path"), + diff_old_text=edit_preview.get("old_text") if edit_preview else None, + diff_new_text=edit_preview.get("new_text") if edit_preview else None, raw_input=metadata.get("raw_input"), ) ) @@ -442,12 +468,14 @@ def _emit_tool_start( def _emit_tool_completed( self, tool_call: ToolCall, + arguments: dict[str, Any], tool_output: dict[str, Any], *, + edit_preview: dict[str, str | None] | None = None, run_id: str, request_id: str, ) -> None: - metadata = _tool_event_metadata(tool_call.name, tool_output) + metadata = _tool_event_metadata(tool_call.name, arguments) if metadata is None: return self._event_sink.tool_call( @@ -461,8 +489,10 @@ def _emit_tool_completed( kind=metadata["kind"], status="completed", path=metadata.get("path"), - content_text=metadata.get("content_text"), - raw_output=metadata.get("raw_output"), + diff_old_text=edit_preview.get("old_text") if edit_preview else None, + diff_new_text=edit_preview.get("new_text") if edit_preview else None, + content_text=None if edit_preview else metadata.get("content_text"), + raw_output=tool_output, ) ) @@ -473,6 +503,8 @@ def _emit_tool_failed( *, run_id: str, request_id: str, + message: str | None = None, + edit_preview: dict[str, str | None] | None = None, ) -> None: metadata = _tool_event_metadata(tool_call.name, arguments) if metadata is None: @@ -488,10 +520,53 @@ def _emit_tool_failed( kind=metadata["kind"], status="failed", path=metadata.get("path"), - content_text=f"{metadata.get('content_text', metadata['title'])} failed", + diff_old_text=edit_preview.get("old_text") if edit_preview else None, + diff_new_text=edit_preview.get("new_text") if edit_preview else None, + content_text=message or f"{metadata.get('content_text', metadata['title'])} failed", ) ) + def _edit_preview( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + *, + run_id: str, + request_id: str, + ) -> dict[str, str | None] | None: + name = tool_call.name + if name not in {"create_file", "write_file", "replace_text"}: + return None + path = str(arguments["path"]) + if name == "create_file": + return {"old_text": None, "new_text": str(arguments.get("content", ""))} + try: + old_text = self._file_tools.read_file( + path=path, + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ).content + except FileNotFoundError: + old_text = "" + if name == "replace_text": + old_fragment = str(arguments.get("old_text", "")) + if old_fragment not in old_text: + raise ValueError("old_text was not found in the file.") + new_text = old_text.replace(old_fragment, str(arguments.get("new_text", "")), 1) + else: + new_text = str(arguments.get("content", "")) + return {"old_text": old_text, "new_text": new_text} + + +def _requires_manual_approval(tool_name: str) -> bool: + return tool_name in { + "create_file", + "write_file", + "replace_text", + "run_command", + } or tool_name.startswith("mcp__") + class FakeOpenAICompatibleProvider: def __init__(self, script: list[dict[str, object]]) -> None: @@ -896,12 +971,26 @@ def handle_prompt( failure_reason=exc.failure_reason, error_message=str(exc), ) - tool_output = { - "status": "denied", - "error": str(exc), - "tool_name": tool_call.name, - "tool_call_id": tool_call.tool_call_id, - } + if exc.failure_reason == "approval_rejected": + tool_output = { + "status": "rejected", + "reason": "user_rejected", + "message": ( + "The user rejected this action. Do not retry this " + "requested change with another mutating tool; explain " + "that no change was made or ask what they prefer instead." + ), + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } + else: + tool_output = { + "status": "denied", + "reason": exc.failure_reason, + "error": str(exc), + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } except PermissionError as exc: tool_output = { "status": "denied", @@ -1431,6 +1520,16 @@ def _tool_event_metadata( fallback_title="Replace text", content_prefix="Replaced text in", ) + if tool_name == "run_command": + argv = values.get("argv") + command = " ".join(str(arg) for arg in argv) if isinstance(argv, (list, tuple)) else "command" + return { + "kind": "execute", + "title": f"Run {command}", + "content_text": f"Run command: {command}", + "raw_input": {"argv": argv, "cwd": values.get("cwd", ".")}, + "raw_output": None, + } if tool_name.startswith("mcp__"): title = f"Call MCP tool {tool_name.removeprefix('mcp__')}" return { diff --git a/src/code4me2_agent/events.py b/src/code4me2_agent/events.py index f4c7c64..bb7a7a5 100644 --- a/src/code4me2_agent/events.py +++ b/src/code4me2_agent/events.py @@ -4,6 +4,16 @@ from typing import Any, Protocol +@dataclass(frozen=True) +class ApprovalDecision: + decision: str + scope: str = "none" + + @property + def accepted(self) -> bool: + return self.decision == "accepted" + + @dataclass(frozen=True) class ToolCallEvent: phase: str @@ -16,6 +26,8 @@ class ToolCallEvent: status: str path: str | None = None content_text: str | None = None + diff_old_text: str | None = None + diff_new_text: str | None = None raw_input: dict[str, Any] | None = None raw_output: dict[str, Any] | None = None diff --git a/tests/code4me2_agent/test_acp_runtime.py b/tests/code4me2_agent/test_acp_runtime.py index baac667..0eedd5e 100644 --- a/tests/code4me2_agent/test_acp_runtime.py +++ b/tests/code4me2_agent/test_acp_runtime.py @@ -4,6 +4,7 @@ import sys from importlib.metadata import version from pathlib import Path +from types import SimpleNamespace from tempfile import TemporaryDirectory from threading import Event from typing import Any @@ -14,12 +15,13 @@ from acp.schema import TextContentBlock from code4me2_agent.adapters import ( + FakeOpenAICompatibleProvider, OpenAICompatibleReactAdapter, ToolCall, ToolRegistry, ToolRegistryError, ) -from code4me2_agent.acp_runtime import create_acp_agent +from code4me2_agent.acp_runtime import AcpSessionEventSink, create_acp_agent from code4me2_agent.acp_updates import AcpUpdateBuilder from code4me2_agent.config import ( AdapterConfig, @@ -28,6 +30,7 @@ ServerAgentConfig, ) from code4me2_agent.echo import EchoPromptResult +from code4me2_agent.events import ApprovalDecision, ToolCallEvent from code4me2_agent.runtime_auth import AcpRuntimeScope from code4me2_agent.telemetry import AgentTelemetryRecorder @@ -111,6 +114,108 @@ async def scenario() -> None: asyncio.run(scenario()) + def test_permission_request_offers_supported_scopes_and_a_summary(self) -> None: + permission = AcpUpdateBuilder().permission_request( + session_id="session-1", + tool_call_id="tool-1", + title="Write file", + kind="edit", + summary="Write file: README.md", + raw_input={"path": "README.md"}, + session_option_name="Allow edits for session", + ) + + self.assertEqual( + ["allow_once", "allow_session", "reject_once"], + [option.option_id for option in permission.options], + ) + self.assertEqual({"path": "README.md"}, permission.tool_call.raw_input) + self.assertEqual("Allow edits for session", permission.options[1].name) + + def test_permission_response_keeps_selected_scope(self) -> None: + class Connection: + async def request_permission(self, **_kwargs): + return None + + class Runner: + calls = 0 + + def run(self, awaitable): + self.calls += 1 + awaitable.close() + return {"outcome": {"outcome": "selected", "optionId": "allow_session"}} + + runner = Runner() + sink = AcpSessionEventSink( + conn=Connection(), + session_id="session-1", + updates=AcpUpdateBuilder(), + telemetry=object(), + async_runner=runner, + ) + + decision = sink.request_approval( + SimpleNamespace(name="write_file", tool_call_id="tool-1"), + {"path": "README.md", "content": "private"}, + ) + + self.assertTrue(decision.accepted) + self.assertEqual("session", decision.scope) + + self.assertTrue( + sink.request_approval( + SimpleNamespace(name="write_file", tool_call_id="tool-2"), + {"path": "other.md", "content": "private"}, + ).accepted + ) + self.assertEqual(1, runner.calls) + + def test_permission_reuses_the_native_edit_diff(self) -> None: + class Connection(_FakeClient): + async def request_permission(self, **kwargs): + self.permission = kwargs + return {"outcome": {"outcome": "selected", "optionId": "allow_once"}} + + class Runner: + def run(self, awaitable): + return asyncio.run(awaitable) + + connection = Connection() + sink = AcpSessionEventSink( + conn=connection, + session_id="session-1", + updates=AcpUpdateBuilder(), + telemetry=object(), + async_runner=Runner(), + ) + sink.tool_call( + ToolCallEvent( + phase="started", + tool_call_id="tool-1", + tool_name="write_file", + run_id="run-1", + request_id="request-1", + title="Update README.md", + kind="edit", + status="pending", + path="README.md", + diff_old_text="before", + diff_new_text="after", + ) + ) + + self.assertTrue( + sink.request_approval( + SimpleNamespace(name="write_file", tool_call_id="tool-1"), + {"path": "README.md", "content": "after"}, + ).accepted + ) + diff = connection.permission["tool_call"].content[0] + self.assertEqual("diff", diff.type) + self.assertEqual("README.md", diff.path) + self.assertEqual("before", diff.old_text) + self.assertEqual("after", diff.new_text) + def test_server_tool_allowlist_hides_and_rejects_write_tools(self) -> None: config = AgentConfig( workspace_root=self.workspace, @@ -187,6 +292,70 @@ def test_denied_tool_call_continues_with_a_tool_result(self) -> None: self.assertEqual("completed", result.run_status) self.assertEqual("I only have read access in this profile.", result.final_response) + def test_rejected_approval_is_explicit_in_the_model_tool_result(self) -> None: + class RejectingSink: + def request_approval(self, *_args): + return ApprovalDecision("rejected") + + def tool_call(self, _event): + return None + + class RecordingProvider(FakeOpenAICompatibleProvider): + def __init__(self): + super().__init__( + [ + { + "tool_calls": [ + { + "id": "create-rejected", + "name": "create_file", + "arguments": {"path": "deneme.txt", "content": "hi"}, + } + ] + }, + {"final_answer": "No change was made."}, + ] + ) + self.messages: list[list[dict[str, Any]]] = [] + + def generate(self, messages): + self.messages.append(messages) + return super().generate(messages) + + config = AgentConfig( + workspace_root=self.workspace, + trace_path=self.workspace / "agent-events.jsonl", + session_id="bootstrap", + tools=["create_file"], + approval_policy="per_step", + ) + provider = RecordingProvider() + adapter = OpenAICompatibleReactAdapter( + config, + telemetry=AgentTelemetryRecorder(config), + tool_registry=ToolRegistry( + file_tools=object(), + command_tools=object(), + event_sink=RejectingSink(), + allowed_tools=config.tools, + approval_policy="per_step", + ), + ) + adapter._provider = lambda: provider + + result = adapter.handle_prompt( + prompt="Create deneme.txt", + run_id="run-1", + request_id="request-1", + message_id=None, + ) + + tool_result = provider.messages[1][-1] + self.assertEqual("tool", tool_result["role"]) + self.assertIn('"reason": "user_rejected"', tool_result["content"]) + self.assertIn("Do not retry", tool_result["content"]) + self.assertEqual("No change was made.", result.final_response) + def test_unsupported_routes_are_method_not_found_even_with_sdk_workaround(self) -> None: async def scenario() -> None: router = build_agent_router(self.agent, use_unstable_protocol=True) diff --git a/tests/code4me2_agent/test_policy_enforcement.py b/tests/code4me2_agent/test_policy_enforcement.py index 47bf66b..654b2f7 100644 --- a/tests/code4me2_agent/test_policy_enforcement.py +++ b/tests/code4me2_agent/test_policy_enforcement.py @@ -1,4 +1,5 @@ from dataclasses import replace +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest @@ -11,6 +12,7 @@ ) from code4me2_agent.config import AgentConfig, MemoryWindowConfig, ServerAgentConfig from code4me2_agent.echo import EchoAgentCore +from code4me2_agent.events import ApprovalDecision from code4me2_agent.runtime_auth import ( AcpBackendAuthorization, AcpRuntimeScope, @@ -21,21 +23,23 @@ class ApprovalSink: - def __init__(self, approved: bool): - self.approved = approved + def __init__(self, decision: ApprovalDecision): + self.decision = decision self.requests = [] + self.events = [] def request_approval(self, tool_call, arguments): self.requests.append((tool_call, arguments)) - return self.approved + return self.decision def tool_call(self, event): - return None + self.events.append(event) -def test_per_step_policy_denies_before_tool_execution(): +def test_per_step_policy_keeps_read_tools_automatic(): file_tools = MagicMock() - sink = ApprovalSink(False) + file_tools.read_file.return_value = {"content": "ok"} + sink = ApprovalSink(ApprovalDecision("rejected")) registry = ToolRegistry( file_tools, MagicMock(), @@ -44,37 +48,66 @@ def test_per_step_policy_denies_before_tool_execution(): approval_policy="per_step", ) - with pytest.raises(ToolRegistryError) as error: - registry.execute( - ToolCall("call-1", "read_file", {"path": "README.md"}), - run_id="run-1", - request_id="request-1", - ) - - assert error.value.failure_reason == "approval_policy_denied" - file_tools.read_file.assert_not_called() - assert len(sink.requests) == 1 + assert registry.execute( + ToolCall("call-1", "read_file", {"path": "README.md"}), + run_id="run-1", + request_id="request-1", + ) == {"content": "ok"} + file_tools.read_file.assert_called_once() + assert sink.requests == [] -def test_per_step_policy_executes_after_allow_once(): +def test_per_step_policy_executes_mutation_after_allow_once(): file_tools = MagicMock() - file_tools.read_file.return_value = {"content": "ok"} + file_tools.read_file.return_value = SimpleNamespace(content="before") + file_tools.write_file.return_value = {"status": "ok"} + sink = ApprovalSink(ApprovalDecision("accepted", "once")) registry = ToolRegistry( file_tools, MagicMock(), - event_sink=ApprovalSink(True), - allowed_tools=frozenset({"read_file"}), + event_sink=sink, + allowed_tools=frozenset({"write_file"}), approval_policy="per_step", ) result = registry.execute( - ToolCall("call-1", "read_file", {"path": "README.md"}), + ToolCall("call-1", "write_file", {"path": "README.md", "content": "ok"}), run_id="run-1", request_id="request-1", ) - assert result == {"content": "ok"} - file_tools.read_file.assert_called_once() + assert result == {"status": "ok"} + file_tools.write_file.assert_called_once() + assert [(event.diff_old_text, event.diff_new_text) for event in sink.events] == [ + ("before", "ok"), + ("before", "ok"), + ] + + +def test_per_step_policy_keeps_rejection_distinct_from_policy_denial(): + file_tools = MagicMock() + sink = ApprovalSink(ApprovalDecision("rejected")) + registry = ToolRegistry( + file_tools, + MagicMock(), + event_sink=sink, + allowed_tools=frozenset({"write_file"}), + approval_policy="per_step", + ) + + with pytest.raises(ToolRegistryError) as error: + registry.execute( + ToolCall("call-1", "write_file", {"path": "README.md", "content": "no"}), + run_id="run-1", + request_id="request-1", + ) + + assert error.value.failure_reason == "approval_rejected" + file_tools.write_file.assert_not_called() + assert [(event.phase, event.status) for event in sink.events] == [ + ("started", "pending"), + ("failed", "failed"), + ] def test_suggestion_only_denies_unknown_mcp_side_effects(): From c1bfc8f97e5fe8626ea934ed7e80261a46e26ae1 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Sun, 13 Sep 2026 19:34:58 +0200 Subject: [PATCH 017/115] fix: backend launch info for agents --- src/backend/routers/agents.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py index 707edbd..0317e26 100644 --- a/src/backend/routers/agents.py +++ b/src/backend/routers/agents.py @@ -116,7 +116,13 @@ def create_agent_task( f"by this session" ) return JSONResponse( - {"task_id": str(existing.task_id)}, status_code=200 + { + "task_id": str(existing.task_id), + "framework_version": existing.framework_version, + "agent_profile": existing.agent_profile, + "model": existing.model, + }, + status_code=200, ) session = crud.get_session_by_id(db, session_id) @@ -168,6 +174,9 @@ def create_agent_task( # The plugin needs the runtime name to know which agent to launch. "framework_version": profile.framework_version, "agent_profile": profile.name, + # This is only the runtime's displayed/requested model. The inference relay + # still enforces the immutable task snapshot on every model call. + "model": task.model, }, status_code=201, ) From 99d5ae15a9ab90d41baac58ea00504bbea728cdd Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 12:16:41 +0200 Subject: [PATCH 018/115] fix: model performance analytics --- .../components/analytics/ModelAnalytics.js | 26 ++++++++++++++++--- src/website/src/pages/Dashboard.js | 8 ++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/website/src/components/analytics/ModelAnalytics.js b/src/website/src/components/analytics/ModelAnalytics.js index 91d7e75..166a60b 100644 --- a/src/website/src/components/analytics/ModelAnalytics.js +++ b/src/website/src/components/analytics/ModelAnalytics.js @@ -3,13 +3,11 @@ import Chart from '../visualization/Chart'; import { getModelComparison } from '../../utils/api'; import './ModelAnalytics.css'; -const ModelAnalytics = () => { +const ModelAnalytics = ({ timeWindow = '30d' }) => { const [modelData, setModelData] = useState(null); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const [selectedModels, setSelectedModels] = useState([]); - const [timeRange, setTimeRange] = useState('30d'); - useEffect(() => { const fetchData = async () => { setIsLoading(true); @@ -20,6 +18,11 @@ const ModelAnalytics = () => { if (selectedModels.length > 0) { params.model_ids = selectedModels.join(','); } + const endTime = new Date(); + const startTime = new Date(endTime); + startTime.setDate(startTime.getDate() - Number.parseInt(timeWindow, 10)); + params.start_time = startTime.toISOString(); + params.end_time = endTime.toISOString(); const response = await getModelComparison(params); if (response.ok) { @@ -36,7 +39,7 @@ const ModelAnalytics = () => { }; fetchData(); - }, [selectedModels, timeRange]); + }, [selectedModels, timeWindow]); const formatModelPerformanceChart = (models) => { if (!models) return []; @@ -86,6 +89,21 @@ const ModelAnalytics = () => { const models = modelData?.data || []; + if (models.length === 0) { + return ( +
+
+

Model Performance Analytics

+

Compare AI model performance, quality metrics, and usage patterns

+
+
+

No model performance data

+

No model generations were recorded in the selected time range.

+
+
+ ); + } + return (
diff --git a/src/website/src/pages/Dashboard.js b/src/website/src/pages/Dashboard.js index 842e5d8..99dde51 100644 --- a/src/website/src/pages/Dashboard.js +++ b/src/website/src/pages/Dashboard.js @@ -15,6 +15,7 @@ import "./Dashboard.css"; const Dashboard = ({ user, onLogout }) => { const [activeView, setActiveView] = useState('overview'); const [timeWindow, setTimeWindow] = useState("7d"); + const [isUrlInitialized, setIsUrlInitialized] = useState(false); const isValidView = (view, user) => { const baseViews = new Set(['overview','usage','models','calibration']); @@ -41,13 +42,16 @@ const Dashboard = ({ user, onLogout }) => { if (tw && ['7d','30d','90d'].includes(tw)) { setTimeWindow(tw); } + setIsUrlInitialized(true); } catch (_) { // ignore URL parse errors + setIsUrlInitialized(true); } }, [user]); // Keep URL in sync with current view and time window so users can deeplink useEffect(() => { + if (!isUrlInitialized) return; try { const url = new URL(window.location.href); if (isValidView(activeView, user)) { @@ -69,7 +73,7 @@ const Dashboard = ({ user, onLogout }) => { } catch (_) { // ignore URL update errors } - }, [activeView, timeWindow, user]); + }, [activeView, timeWindow, user, isUrlInitialized]); const renderActiveView = () => { switch (activeView) { @@ -81,7 +85,7 @@ const Dashboard = ({ user, onLogout }) => { case 'usage': return ; case 'models': - return ; + return ; case 'calibration': return ; case 'studies': From ad3555e26626fd1df3091c10a6f4a93bb7fbd07f Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 13:43:53 +0200 Subject: [PATCH 019/115] feat: add agent telemetry dashboard --- src/backend/routers/analytics/__init__.py | 3 +- src/backend/routers/analytics/agents.py | 449 ++++++++++++++++++ .../components/analytics/AgentAnalytics.css | 80 ++++ .../components/analytics/AgentAnalytics.js | 232 +++++++++ .../analytics/AnalyticsNavigation.js | 6 + src/website/src/pages/Dashboard.js | 24 +- src/website/src/utils/api.js | 27 ++ 7 files changed, 810 insertions(+), 11 deletions(-) create mode 100644 src/backend/routers/analytics/agents.py create mode 100644 src/website/src/components/analytics/AgentAnalytics.css create mode 100644 src/website/src/components/analytics/AgentAnalytics.js diff --git a/src/backend/routers/analytics/__init__.py b/src/backend/routers/analytics/__init__.py index 810a5bd..b184914 100644 --- a/src/backend/routers/analytics/__init__.py +++ b/src/backend/routers/analytics/__init__.py @@ -13,12 +13,13 @@ """ from fastapi import APIRouter -from . import usage, models, calibration, studies, overview +from . import agents, usage, models, calibration, studies, overview router = APIRouter() # Include all analytics sub-routers router.include_router(usage.router, prefix="/usage", tags=["Usage Analytics"]) +router.include_router(agents.router, prefix="/agents", tags=["Agent Analytics"]) router.include_router(models.router, prefix="/models", tags=["Model Analytics"]) router.include_router(calibration.router, prefix="/calibration", tags=["Model Calibration"]) router.include_router(studies.router, prefix="/studies", tags=["A/B Testing"]) diff --git a/src/backend/routers/analytics/agents.py b/src/backend/routers/analytics/agents.py new file mode 100644 index 0000000..bb10b70 --- /dev/null +++ b/src/backend/routers/analytics/agents.py @@ -0,0 +1,449 @@ +"""Privacy-scoped aggregate analytics for agent invocations.""" + +from datetime import datetime, timedelta, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import text + +from App import App +from backend.Responses import JsonResponseWithStatus +from .auth_utils import AuthenticatedUser, get_current_user + +router = APIRouter() +_TIME_WINDOWS = {"7d": 7, "30d": 30, "90d": 90} + + +def _as_number(value, integer: bool = False): + if value is None: + return 0 if integer else 0.0 + return int(value) if integer else float(value) + + +def _iso(value): + return value.isoformat() if value is not None else None + + +@router.get("/overview") +def get_agent_overview( + time_window: str = Query("7d", description="7d, 30d, or 90d"), + framework: Optional[str] = Query(None), + model: Optional[str] = Query(None), + profile: Optional[str] = Query(None), + user_id: Optional[str] = Query(None, description="Admin-only user filter"), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Aggregate task/event telemetry without returning content payloads.""" + days = _TIME_WINDOWS.get(time_window) + if days is None: + raise HTTPException(status_code=400, detail="time_window must be 7d, 30d, or 90d") + + db = app.get_db_session() + try: + end_time = datetime.now(timezone.utc) + start_time = end_time - timedelta(days=days) + params = {"start_time": start_time, "end_time": end_time} + conditions = ["t.created_at BETWEEN :start_time AND :end_time"] + + if current_user.is_admin and user_id: + params["user_id"] = user_id + conditions.append("CAST(t.owner_user_id AS TEXT) = :user_id") + elif not current_user.is_admin: + params["user_id"] = str(current_user.user_id) + conditions.append("t.owner_user_id = CAST(:user_id AS UUID)") + + if framework: + params["framework"] = framework + conditions.append("t.framework_version = :framework") + if model: + params["model"] = model + conditions.append("t.model = :model") + if profile: + params["profile"] = profile + conditions.append("t.agent_profile = :profile") + + where = " AND ".join(conditions) + + task_query = f""" + SELECT + COUNT(*) AS total_tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed_tasks, + COUNT(*) FILTER (WHERE t.status IN ('pending', 'running')) AS open_tasks, + COALESCE(AVG(t.total_steps), 0) AS avg_steps, + COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) + AS avg_task_duration_ms, + COALESCE(SUM(t.input_tokens), 0) AS input_tokens, + COALESCE(SUM(t.output_tokens), 0) AS output_tokens + FROM agent_task t + WHERE {where} + """ + task = db.execute(text(task_query), params).one() + + events_query = f""" + SELECT + COUNT(*) FILTER (WHERE e.event_type = 'model_request') AS model_requests, + COUNT(*) FILTER (WHERE e.event_type = 'model_call') AS model_calls, + COUNT(*) FILTER (WHERE e.event_type = 'tool_request') AS tool_requests, + COUNT(*) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, + COUNT(*) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures, + COALESCE(AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call'), 0) + AS avg_model_latency_ms, + COALESCE(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY e.latency_ms) + FILTER (WHERE e.event_type = 'model_call'), 0) + AS p95_model_latency_ms, + COALESCE(SUM(e.prompt_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) + AS provider_input_tokens, + COALESCE(SUM(e.completion_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) + AS model_output_tokens, + COUNT(*) FILTER (WHERE e.event_type = 'model_call' AND e.upstream_status = 429) + AS rate_limit_retries, + COUNT(*) FILTER (WHERE e.event_type = 'model_call' + AND e.upstream_status >= 200 AND e.upstream_status < 400) + AS successful_model_calls, + COALESCE(SUM( + CASE WHEN e.event_type = 'model_call' + THEN COALESCE((NULLIF(e.extra_json, '')::jsonb ->> 'tool_schema_bytes')::bigint, 0) + ELSE 0 END + ), 0) AS tool_schema_bytes, + COALESCE(SUM(e.context_window_size_bytes) + FILTER (WHERE e.event_type = 'model_call'), 0) + AS conversation_context_bytes, + COALESCE(SUM(e.tool_result_length) + FILTER (WHERE e.event_type = 'tool_call'), 0) + AS tool_result_bytes, + COALESCE(SUM(e.total_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) + AS event_tokens + FROM agent_event e + JOIN agent_task t ON t.task_id = e.task_id + WHERE {where} + """ + events = db.execute(text(events_query), params).one() + + edits_query = f""" + SELECT + COUNT(ed.edit_id) AS total_edits, + COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS TRUE) AS accepted_edits, + COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS FALSE) AS rejected_edits + FROM agent_task t + LEFT JOIN agent_edit ed ON ed.task_id = t.task_id + WHERE {where} + """ + edits = db.execute(text(edits_query), params).one() + + profiles_query = f""" + SELECT + COALESCE(t.agent_profile, 'unknown') AS profile_name, + COALESCE(t.framework_version, 'unknown') AS framework_version, + COALESCE(t.model, 'unknown') AS model, + COUNT(*) AS tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE(AVG(t.total_steps), 0) AS avg_steps + FROM agent_task t + WHERE {where} + GROUP BY 1, 2, 3 + ORDER BY tasks DESC, profile_name + """ + profile_rows = db.execute(text(profiles_query), params).fetchall() + + tools_query = f""" + SELECT + COALESCE(e.tool_name, 'unknown') AS tool_name, + COUNT(*) FILTER (WHERE e.event_type = 'tool_call') AS calls, + COUNT(*) FILTER (WHERE e.event_type = 'tool_failed') AS failures, + COALESCE(AVG(e.latency_ms), 0) AS avg_latency_ms + FROM agent_event e + JOIN agent_task t ON t.task_id = e.task_id + WHERE {where} AND e.event_type IN ('tool_call', 'tool_failed') + GROUP BY 1 + ORDER BY calls DESC, tool_name + LIMIT 20 + """ + tool_rows = db.execute(text(tools_query), params).fetchall() + + runs_query = f""" + SELECT + t.task_id, + t.agent_profile, + t.framework_version, + t.model, + t.status, + t.total_steps, + t.created_at, + t.started_at, + t.completed_at, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, + COUNT(e.event_id) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures + FROM agent_task t + LEFT JOIN agent_event e ON e.task_id = t.task_id + WHERE {where} + GROUP BY t.task_id + ORDER BY t.created_at DESC + LIMIT 25 + """ + run_rows = db.execute(text(runs_query), params).fetchall() + + trend_query = f""" + SELECT + DATE_TRUNC('day', t.created_at) AS day, + COUNT(*) AS runs, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE( + AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), + 0 + ) AS avg_duration_ms + FROM agent_task t + WHERE {where} + GROUP BY 1 + ORDER BY 1 + """ + trend_rows = db.execute(text(trend_query), params).fetchall() + + latency_query = f""" + SELECT + CASE + WHEN e.latency_ms < 250 THEN '<250ms' + WHEN e.latency_ms < 500 THEN '250-499ms' + WHEN e.latency_ms < 1000 THEN '500-999ms' + WHEN e.latency_ms < 2000 THEN '1-2s' + ELSE '2s+' + END AS bucket, + CASE + WHEN e.latency_ms < 250 THEN 1 + WHEN e.latency_ms < 500 THEN 2 + WHEN e.latency_ms < 1000 THEN 3 + WHEN e.latency_ms < 2000 THEN 4 + ELSE 5 + END AS bucket_order, + COUNT(*) AS calls + FROM agent_event e + JOIN agent_task t ON t.task_id = e.task_id + WHERE {where} AND e.event_type = 'model_call' AND e.latency_ms IS NOT NULL + GROUP BY 1, 2 + ORDER BY bucket_order + """ + latency_rows = db.execute(text(latency_query), params).fetchall() + + event_types_query = f""" + SELECT e.event_type, COUNT(*) AS events + FROM agent_event e + JOIN agent_task t ON t.task_id = e.task_id + WHERE {where} + GROUP BY e.event_type + ORDER BY events DESC, e.event_type + """ + event_type_rows = db.execute(text(event_types_query), params).fetchall() + + errors_query = f""" + SELECT + CASE + WHEN e.upstream_status IS NOT NULL AND e.upstream_status >= 400 + THEN 'upstream_' || e.upstream_status::text + ELSE e.event_type + END AS reason, + COUNT(*) AS events + FROM agent_event e + JOIN agent_task t ON t.task_id = e.task_id + WHERE {where} + AND (e.event_type IN ('error', 'tool_failed', 'tool_denied') + OR e.upstream_status >= 400) + GROUP BY 1 + ORDER BY events DESC, reason + """ + error_rows = db.execute(text(errors_query), params).fetchall() + + total_tasks = _as_number(task.total_tasks, integer=True) + decided_edits = _as_number(edits.accepted_edits, integer=True) + _as_number( + edits.rejected_edits, integer=True + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "filters": {"time_window": time_window, "framework": framework, "model": model, "profile": profile}, + "summary": { + "total_tasks": total_tasks, + "completed_tasks": _as_number(task.completed_tasks, integer=True), + "failed_tasks": _as_number(task.failed_tasks, integer=True), + "open_tasks": _as_number(task.open_tasks, integer=True), + "completion_rate": _as_number(task.completed_tasks, integer=True) / max(total_tasks, 1), + "avg_steps": _as_number(task.avg_steps), + "avg_task_duration_ms": _as_number(task.avg_task_duration_ms), + "input_tokens": _as_number(task.input_tokens, integer=True), + "output_tokens": _as_number(task.output_tokens, integer=True), + "model_requests": _as_number(events.model_requests, integer=True), + "model_calls": _as_number(events.model_calls, integer=True), + "tool_requests": _as_number(events.tool_requests, integer=True), + "tool_calls": _as_number(events.tool_calls, integer=True), + "failures": _as_number(events.failures, integer=True), + "avg_model_latency_ms": _as_number(events.avg_model_latency_ms), + "p95_model_latency_ms": _as_number(events.p95_model_latency_ms), + "provider_input_tokens": _as_number(events.provider_input_tokens, integer=True), + "model_output_tokens": _as_number(events.model_output_tokens, integer=True), + "successful_model_calls": _as_number(events.successful_model_calls, integer=True), + "rate_limit_retries": _as_number(events.rate_limit_retries, integer=True), + # These are estimates based on UTF-8 bytes / 4, not provider billing tokens. + "tool_schema_tokens_estimated": _as_number(events.tool_schema_bytes, integer=True) // 4, + "conversation_tokens_estimated": _as_number(events.conversation_context_bytes, integer=True) // 4, + "tool_result_tokens_estimated": _as_number(events.tool_result_bytes, integer=True) // 4, + "event_tokens": _as_number(events.event_tokens, integer=True), + "total_edits": _as_number(edits.total_edits, integer=True), + "edit_acceptance_rate": _as_number(edits.accepted_edits, integer=True) / max(decided_edits, 1), + }, + "profiles": [ + { + "profile_name": row.profile_name, + "framework_version": row.framework_version, + "model": row.model, + "tasks": _as_number(row.tasks, integer=True), + "completed": _as_number(row.completed, integer=True), + "failed": _as_number(row.failed, integer=True), + "completion_rate": _as_number(row.completed, integer=True) / max(_as_number(row.tasks, integer=True), 1), + "avg_steps": _as_number(row.avg_steps), + } + for row in profile_rows + ], + "tools": [ + { + "tool_name": row.tool_name, + "calls": _as_number(row.calls, integer=True), + "failures": _as_number(row.failures, integer=True), + "avg_latency_ms": _as_number(row.avg_latency_ms), + } + for row in tool_rows + ], + "recent_runs": [ + { + "task_id": str(row.task_id), + "profile_name": row.agent_profile, + "framework_version": row.framework_version, + "model": row.model, + "status": row.status, + "steps": _as_number(row.total_steps, integer=True), + "created_at": _iso(row.created_at), + "started_at": _iso(row.started_at), + "completed_at": _iso(row.completed_at), + "model_calls": _as_number(row.model_calls, integer=True), + "tool_calls": _as_number(row.tool_calls, integer=True), + "failures": _as_number(row.failures, integer=True), + } + for row in run_rows + ], + "trend": [ + { + "day": _iso(row.day), + "runs": _as_number(row.runs, integer=True), + "completed": _as_number(row.completed, integer=True), + "failed": _as_number(row.failed, integer=True), + "avg_duration_ms": _as_number(row.avg_duration_ms), + } + for row in trend_rows + ], + "latency_distribution": [ + {"bucket": row.bucket, "calls": _as_number(row.calls, integer=True)} + for row in latency_rows + ], + "event_types": [ + {"event_type": row.event_type, "events": _as_number(row.events, integer=True)} + for row in event_type_rows + ], + "errors": [ + {"reason": row.reason, "events": _as_number(row.events, integer=True)} + for row in error_rows + ], + }, + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + raise HTTPException(status_code=500, detail=f"Error retrieving agent analytics: {error}") from error + finally: + db.close() + + +@router.get("/runs/{task_id}") +def get_agent_run_detail( + task_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return structural event details for one authorized run, never content.""" + db = app.get_db_session() + try: + params = {"task_id": task_id} + owner_filter = "" + if not current_user.is_admin: + params["user_id"] = str(current_user.user_id) + owner_filter = "AND CAST(t.owner_user_id AS TEXT) = :user_id" + + task_query = f""" + SELECT t.task_id, t.agent_profile, t.framework_version, t.model, + t.status, t.total_steps, t.created_at, t.started_at, t.completed_at + FROM agent_task t + WHERE CAST(t.task_id AS TEXT) = :task_id {owner_filter} + """ + task = db.execute(text(task_query), params).one_or_none() + if task is None: + raise HTTPException(status_code=404, detail="Agent run not found") + + events_query = """ + SELECT event_index, event_type, source, latency_ms, created_at, + occurred_at, span_id, parent_span_id, model, tool_name, + prompt_tokens, completion_tokens, total_tokens, + finish_reason, upstream_status, step_index + FROM agent_event + WHERE task_id = CAST(:task_id AS UUID) + ORDER BY event_index + """ + events = db.execute(text(events_query), {"task_id": task_id}).fetchall() + + return JsonResponseWithStatus( + status_code=200, + content={ + "run": { + "task_id": str(task.task_id), + "profile_name": task.agent_profile, + "framework_version": task.framework_version, + "model": task.model, + "status": task.status, + "steps": _as_number(task.total_steps, integer=True), + "created_at": _iso(task.created_at), + "started_at": _iso(task.started_at), + "completed_at": _iso(task.completed_at), + }, + "events": [ + { + "event_index": _as_number(row.event_index, integer=True), + "event_type": row.event_type, + "source": row.source, + "latency_ms": _as_number(row.latency_ms), + "created_at": _iso(row.created_at), + "occurred_at": _iso(row.occurred_at), + "span_id": row.span_id, + "parent_span_id": str(row.parent_span_id) if row.parent_span_id else None, + "model": row.model, + "tool_name": row.tool_name, + "prompt_tokens": _as_number(row.prompt_tokens, integer=True), + "completion_tokens": _as_number(row.completion_tokens, integer=True), + "total_tokens": _as_number(row.total_tokens, integer=True), + "finish_reason": row.finish_reason, + "upstream_status": row.upstream_status, + "step_index": row.step_index, + } + for row in events + ], + }, + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + raise HTTPException(status_code=500, detail=f"Error retrieving agent run: {error}") from error + finally: + db.close() \ No newline at end of file diff --git a/src/website/src/components/analytics/AgentAnalytics.css b/src/website/src/components/analytics/AgentAnalytics.css new file mode 100644 index 0000000..c092deb --- /dev/null +++ b/src/website/src/components/analytics/AgentAnalytics.css @@ -0,0 +1,80 @@ +.agent-analytics { max-width: 1200px; margin: 0 auto; } +.agent-page-header { display: flex; justify-content: space-between; gap: 20px; align-items: end; margin: 0 0 24px; } +.agent-page-header h2 { margin: 4px 0 8px; color: var(--text-color); font-size: 28px; } +.agent-page-header p { margin: 0; color: var(--text-secondary); } +.agent-eyebrow { color: var(--primary-color); font-size: 11px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; } +.agent-window { padding: 8px 12px; border: 1px solid var(--border-color); border-radius: 4px; color: var(--text-secondary); font-size: 12px; white-space: nowrap; } +.agent-controls { display: flex; align-items: end; gap: 14px; flex-wrap: wrap; padding: 14px 16px; margin-bottom: 18px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); } +.agent-controls label { display: flex; flex-direction: column; gap: 5px; color: var(--text-secondary); font-size: 12px; font-weight: 600; } +.agent-controls select { min-width: 170px; padding: 8px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--card-background); color: var(--text-color); } +.agent-clear { padding: 8px 12px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--secondary-color); color: var(--text-color); cursor: pointer; } +.agent-kpis { display: grid; grid-template-columns: repeat(6, minmax(120px, 1fr)); gap: 10px; margin-bottom: 18px; } +.agent-kpis > div { min-height: 100px; padding: 14px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 2px 8px var(--shadow-color); } +.agent-kpis span, .agent-kpis small { display: block; color: var(--text-secondary); font-size: 11px; } +.agent-kpis strong { display: block; margin: 8px 0 4px; color: var(--text-color); font-size: 23px; } +.agent-token-panel { margin-bottom: 18px; } +.agent-token-grid { display: grid; grid-template-columns: repeat(6, minmax(120px, 1fr)); gap: 10px; } +.agent-token-grid > div { min-height: 82px; padding: 12px; border: 1px solid var(--border-color); border-radius: 6px; background: var(--secondary-color); } +.agent-token-grid span, .agent-token-grid small { display: block; color: var(--text-secondary); font-size: 11px; } +.agent-token-grid strong { display: block; margin: 8px 0 3px; color: var(--text-color); font-size: 19px; } +.agent-layout { display: grid; grid-template-columns: minmax(0, 1.6fr) minmax(280px, 1fr); gap: 18px; } +.agent-analysis-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; margin-bottom: 18px; } +.agent-panel { min-width: 0; padding: 18px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 2px 8px var(--shadow-color); } +.agent-wide { grid-column: 1 / -1; } +.agent-panel-heading { display: flex; justify-content: space-between; align-items: end; gap: 12px; margin-bottom: 14px; } +.agent-panel h3 { margin: 3px 0 0; color: var(--text-color); font-size: 17px; } +.agent-muted { color: var(--text-secondary); font-size: 11px; } +.agent-table-scroll { overflow-x: auto; } +.agent-runs-scroll { max-height: 360px; overflow-y: auto; } +.agent-table { width: 100%; border-collapse: collapse; color: var(--text-color); font-size: 12px; } +.agent-table th { padding: 10px 8px; background: var(--secondary-color); color: var(--text-secondary); text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .04em; white-space: nowrap; } +.agent-table td { padding: 11px 8px; border-bottom: 1px solid var(--border-color); white-space: nowrap; } +.agent-table td small { display: block; margin-top: 3px; color: var(--text-secondary); font-size: 10px; } +.agent-rate { font-weight: 700; } +.agent-status { display: inline-block; padding: 3px 7px; border-radius: 4px; font-size: 10px; font-weight: 700; text-transform: uppercase; } +.agent-status.done { background: rgba(21, 128, 61, .14); color: #15803d; } +.agent-status.failed { background: rgba(194, 65, 12, .14); color: #c2410c; } +.agent-status.pending, .agent-status.running { background: rgba(202, 138, 4, .14); color: #a16207; } +.agent-tool-list { display: grid; gap: 8px; } +.agent-tool { padding: 10px 0; border-bottom: 1px solid var(--border-color); } +.agent-tool > div { display: flex; justify-content: space-between; gap: 10px; } +.agent-tool > div + div { margin-top: 5px; color: var(--text-secondary); font-size: 11px; } +.agent-bad { color: #c2410c; } +.agent-good { color: #15803d; } +.agent-empty { margin: 0; color: var(--text-secondary); font-size: 13px; } +.agent-panel .chart-container { height: 210px; margin: 0; } +.agent-distribution { display: grid; gap: 13px; padding: 8px 0; } +.agent-distribution-row { display: grid; grid-template-columns: 76px 1fr 34px; gap: 8px; align-items: center; color: var(--text-secondary); font-size: 11px; } +.agent-distribution-row > div { height: 10px; overflow: hidden; border-radius: 3px; background: var(--secondary-color); } +.agent-distribution-row i { display: block; height: 100%; border-radius: inherit; background: #0f766e; } +.agent-distribution-row strong { color: var(--text-color); text-align: right; } +.agent-signal-list { display: grid; gap: 8px; } +.agent-signal-list div { display: flex; justify-content: space-between; gap: 12px; padding: 8px 0; border-bottom: 1px solid var(--border-color); color: var(--text-secondary); font-size: 12px; } +.agent-signal-list strong { color: var(--text-color); } +.agent-error-list strong { color: #c2410c; } +.agent-inspect, .agent-close { padding: 5px 9px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--secondary-color); color: var(--text-color); cursor: pointer; font-size: 11px; } +.agent-inspector { margin-top: 18px; padding: 18px; border: 1px solid var(--primary-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 4px 16px var(--shadow-color); } +.agent-inspector-heading { display: flex; justify-content: space-between; align-items: end; gap: 12px; margin-bottom: 14px; } +.agent-inspector-heading .agent-panel-heading { margin-bottom: 0; } +.agent-event-scroll { max-height: 420px; overflow-y: auto; padding: 4px 6px 4px 0; } +.agent-event-graph { display: grid; gap: 6px; min-width: 420px; } +.agent-graph-row { position: relative; display: flex; gap: 8px; min-height: 48px; } +.agent-graph-rail { position: relative; width: 12px; flex: 0 0 12px; } +.agent-graph-rail::before { position: absolute; top: 0; bottom: -6px; left: 5px; width: 2px; background: var(--border-color); content: ""; } +.agent-graph-rail::after { position: absolute; top: 18px; left: 1px; width: 10px; height: 10px; border: 2px solid var(--card-background); border-radius: 50%; background: #0f766e; box-shadow: 0 0 0 1px #0f766e; content: ""; } +.agent-graph-rail.tool::after { background: #d97706; box-shadow: 0 0 0 1px #d97706; } +.agent-graph-rail.failure::after { background: #c2410c; box-shadow: 0 0 0 1px #c2410c; } +.agent-graph-node { display: flex; justify-content: space-between; gap: 14px; width: 100%; padding: 8px 10px; border: 1px solid rgba(15, 118, 110, .25); border-radius: 6px; background: rgba(15, 118, 110, .06); } +.agent-graph-node.tool { border-color: rgba(217, 119, 6, .3); background: rgba(217, 119, 6, .06); } +.agent-graph-node.failure { border-color: rgba(194, 65, 12, .4); background: rgba(194, 65, 12, .07); } +.agent-graph-node-main { min-width: 0; } +.agent-graph-node-main strong, .agent-graph-node-main small { display: block; } +.agent-graph-node-main strong { color: var(--text-color); font-size: 12px; } +.agent-graph-node-main small { margin-top: 3px; overflow: hidden; color: var(--text-secondary); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; } +.agent-graph-node-stats { display: flex; flex: 0 0 auto; align-items: center; gap: 8px; color: var(--text-secondary); font-size: 10px; white-space: nowrap; } +.agent-run-meta { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; color: var(--text-secondary); font-size: 12px; } +.agent-run-meta > span:not(.agent-status) { padding: 5px 8px; border-radius: 4px; background: var(--secondary-color); } +.agent-event-index { color: var(--primary-color); font-variant-numeric: tabular-nums; } +.agent-error-text { color: #c2410c; } +@media (max-width: 1050px) { .agent-kpis, .agent-token-grid { grid-template-columns: repeat(3, minmax(140px, 1fr)); } } +@media (max-width: 680px) { .agent-page-header, .agent-panel-heading { align-items: start; flex-direction: column; } .agent-kpis, .agent-layout, .agent-analysis-grid { grid-template-columns: 1fr; } .agent-wide { grid-column: auto; } .agent-controls select { width: 100%; } .agent-event-graph { min-width: 0; } .agent-graph-node { align-items: start; flex-direction: column; gap: 5px; } .agent-graph-node-stats { flex-wrap: wrap; } } diff --git a/src/website/src/components/analytics/AgentAnalytics.js b/src/website/src/components/analytics/AgentAnalytics.js new file mode 100644 index 0000000..810d230 --- /dev/null +++ b/src/website/src/components/analytics/AgentAnalytics.js @@ -0,0 +1,232 @@ +import React, { useEffect, useState } from "react"; +import Chart from "../visualization/Chart"; +import { getAgentOverview, getAgentRunDetail } from "../../utils/api"; +import "./AgentAnalytics.css"; + +const number = (value) => Number(value || 0).toLocaleString(); +const percent = (value) => `${(Number(value || 0) * 100).toFixed(1)}%`; +const milliseconds = (value) => `${Math.round(Number(value || 0))}ms`; + +const mergeOptions = (previous, values) => [...new Set([...previous, ...values])].sort(); + +const FilterSelect = ({ label, value, options, onChange, allLabel }) => ( + +); + +const PanelHeading = ({ eyebrow, title, hint }) => ( +
+
{eyebrow}

{title}

+ {hint && {hint}} +
+); + +const KpiCard = ({ label, value, detail }) => ( +
{label}{value}{detail}
+); + +const eventDepth = (event, spanIndexes, events) => { + if (!event.parent_span_id) return 0; + let parentIndex = spanIndexes.get(event.parent_span_id); + let depth = 0; + while (parentIndex !== undefined && depth < 4) { + depth += 1; + const parent = events[parentIndex]; + parentIndex = spanIndexes.get(parent?.parent_span_id); + } + return depth; +}; + +const EventGraph = ({ events }) => { + const spanIndexes = new Map(events.map((event, index) => [event.span_id, index])); + return ( +
+
+ {events.map((event) => { + const depth = eventDepth(event, spanIndexes, events); + const isTool = event.event_type.includes("tool"); + const isFailure = event.event_type.includes("failed") || event.upstream_status >= 400; + return ( +
+
+
+
+ {event.event_index} + {event.event_type} + {event.source || "unknown source"}{event.tool_name ? ` · ${event.tool_name}` : event.model ? ` · ${event.model}` : ""} +
+
+ {event.latency_ms ? {milliseconds(event.latency_ms)} : null} + {event.total_tokens ? {number(event.total_tokens)} tokens : null} + {event.upstream_status ? HTTP {event.upstream_status} : null} +
+
+
+ ); + })} +
+
+ ); +}; + +const AgentAnalytics = ({ timeWindow = "7d" }) => { + const [data, setData] = useState(null); + const [framework, setFramework] = useState(""); + const [model, setModel] = useState(""); + const [profile, setProfile] = useState(""); + const [filterOptions, setFilterOptions] = useState({ + frameworks: [], + models: [], + profiles: [], + }); + const [selectedRun, setSelectedRun] = useState(null); + const [runDetail, setRunDetail] = useState(null); + const [isRunLoading, setIsRunLoading] = useState(false); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + const load = async () => { + setIsLoading(true); + setError(null); + const params = { time_window: timeWindow }; + if (framework) params.framework = framework; + if (model) params.model = model; + if (profile) params.profile = profile; + const response = await getAgentOverview(params); + if (cancelled) return; + if (response.ok) { + setData(response.data); + const nextProfiles = response.data.profiles || []; + setFilterOptions((previous) => ({ + frameworks: mergeOptions(previous.frameworks, nextProfiles.map((item) => item.framework_version)), + models: mergeOptions(previous.models, nextProfiles.map((item) => item.model)), + profiles: mergeOptions(previous.profiles, nextProfiles.map((item) => item.profile_name)), + })); + } else setError(response.error); + setIsLoading(false); + }; + load().catch((loadError) => { + console.error("Agent telemetry error:", loadError); + setError("Failed to load agent telemetry"); + setIsLoading(false); + }); + return () => { cancelled = true; }; + }, [timeWindow, framework, model, profile]); + + const openRun = async (taskId) => { + setSelectedRun(taskId); + setIsRunLoading(true); + const response = await getAgentRunDetail(taskId); + setRunDetail(response.ok ? response.data : { error: response.error }); + setIsRunLoading(false); + }; + + if (isLoading) { + return

Loading Agent Telemetry...

; + } + + if (error) { + return

Failed to load agent telemetry

{error}

; + } + + const summary = data?.summary || {}; + const profiles = data?.profiles || []; + const tools = data?.tools || []; + const runs = data?.recent_runs || []; + const trend = data?.trend || []; + const latencyDistribution = data?.latency_distribution || []; + const maxLatencyCalls = Math.max(...latencyDistribution.map((item) => item.calls), 1); + const eventTypes = data?.event_types || []; + const errors = data?.errors || []; + const frameworks = filterOptions.frameworks; + const models = filterOptions.models; + const availableProfiles = filterOptions.profiles; + + return ( +
+
+
+ Operational telemetry +

Agent Performance

+

Understand how agents run, where they spend time, and where execution fails.

+
+
Last {timeWindow.replace("d", " days")}
+
+ +
+ + + + {(framework || model || profile) && } +
+ +
+ + + + + + +
+ +
+ +
+
Provider input tokens{number(summary.provider_input_tokens)}reported by model provider
+
Conversation context~{number(summary.conversation_tokens_estimated)}estimated from message bytes
+
Tool schema~{number(summary.tool_schema_tokens_estimated)}estimated from schema bytes
+
Tool results~{number(summary.tool_result_tokens_estimated)}estimated from result bytes
+
Model output{number(summary.model_output_tokens)}reported completion tokens
+
Successful calls{number(summary.successful_model_calls)}{number(summary.rate_limit_retries)} rate-limit retries
+
+
+ +
+
+ + {trend.length ? ({ label: item.day.slice(5, 10), value: item.runs }))} title="Runs per day" color="#0f766e" xKey="label" xType="category" /> :

No trend data in this time range.

} +
+
+ + {latencyDistribution.length ?
{latencyDistribution.map((item) =>
{item.bucket}
{number(item.calls)}
)}
:

No latency data in this time range.

} +
+
+ + {eventTypes.length ?
{eventTypes.slice(0, 8).map((item) =>
{item.event_type}{number(item.events)}
)}
:

No event data in this time range.

} +
+
+ + {errors.length ?
{errors.map((item) =>
{item.reason}{number(item.events)}
)}
:

No recorded failures in this time range.

} +
+
+ +
+
+ + {profiles.length ?
{profiles.map((item) => )}
ProfileRuntimeModelRunsCompletionFailedAvg steps
{item.profile_name}{item.framework_version}{item.model}{number(item.tasks)}{percent(item.completion_rate)}{number(item.failed)}{Number(item.avg_steps || 0).toFixed(1)}
:

No profile telemetry in this time range.

} +
+ +
+ + {tools.length ?
{tools.map((item) =>
{item.tool_name}{number(item.calls)} calls
{milliseconds(item.avg_latency_ms)}{number(item.failures)} failed
)}
:

No tool telemetry in this time range.

} +
+ +
+ + {runs.length ?
{runs.map((run) => )}
StatusProfileModelCreatedStepsModel callsTool callsFailures
{run.status}{run.profile_name || "unknown"}{run.framework_version}{run.model}{run.created_at ? new Date(run.created_at).toLocaleString() : "n/a"}{number(run.steps)}{number(run.model_calls)}{number(run.tool_calls)}{number(run.failures)}
:

No agent runs in this time range.

} +
+
+ + {selectedRun &&
{isRunLoading &&

Loading run events...

}{runDetail?.error &&

{runDetail.error}

}{runDetail?.run &&
{runDetail.run.profile_name}{runDetail.run.model}{runDetail.run.status}
}{runDetail?.events?.length > 0 && }{runDetail?.events?.length === 0 && !isRunLoading &&

This run has no persisted events.

}
} +
+ ); +}; + +export default AgentAnalytics; diff --git a/src/website/src/components/analytics/AnalyticsNavigation.js b/src/website/src/components/analytics/AnalyticsNavigation.js index 22c1edd..e82626d 100644 --- a/src/website/src/components/analytics/AnalyticsNavigation.js +++ b/src/website/src/components/analytics/AnalyticsNavigation.js @@ -21,6 +21,12 @@ const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { icon: '🤖', description: 'AI model comparison and quality metrics' }, + { + id: 'agents', + label: 'Agent Telemetry', + icon: '⚙️', + description: 'Runs, tools, latency, failures and outcomes' + }, { id: 'calibration', label: 'Model Calibration', diff --git a/src/website/src/pages/Dashboard.js b/src/website/src/pages/Dashboard.js index 99dde51..52bf442 100644 --- a/src/website/src/pages/Dashboard.js +++ b/src/website/src/pages/Dashboard.js @@ -5,6 +5,7 @@ import AnalyticsNavigation from "../components/analytics/AnalyticsNavigation"; import OverviewDashboard from "../components/analytics/OverviewDashboard"; import UsageAnalytics from "../components/analytics/UsageAnalytics"; import ModelAnalytics from "../components/analytics/ModelAnalytics"; +import AgentAnalytics from "../components/analytics/AgentAnalytics"; import CalibrationAnalytics from "../components/analytics/CalibrationAnalytics"; import StudyManagement from "../components/analytics/StudyManagement"; import ConfigManagement from "../components/analytics/ConfigManagement"; @@ -12,21 +13,22 @@ import AgentProfiles from "./AgentProfiles"; import AgentAssignments from "./AgentAssignments"; import "./Dashboard.css"; +const BASE_VIEWS = new Set(["overview", "usage", "models", "agents", "calibration"]); + +const isValidView = (view, user) => { + if (BASE_VIEWS.has(view)) return true; + if (view === "studies") return !!user?.is_admin; + if (view === "configs") return !!user?.is_admin; + if (view === "agent-profiles") return !!user?.is_admin; + if (view === "agent-assignments") return !!user?.is_admin; + return false; +}; + const Dashboard = ({ user, onLogout }) => { const [activeView, setActiveView] = useState('overview'); const [timeWindow, setTimeWindow] = useState("7d"); const [isUrlInitialized, setIsUrlInitialized] = useState(false); - const isValidView = (view, user) => { - const baseViews = new Set(['overview','usage','models','calibration']); - if (baseViews.has(view)) return true; - if (view === 'studies') return !!user?.is_admin; - if (view === 'configs') return !!user?.is_admin; - if (view === 'agent-profiles') return !!user?.is_admin; - if (view === 'agent-assignments') return !!user?.is_admin; - return false; - }; - // Initialize activeView/timeWindow from URL (query ?view=... or hash #studies) useEffect(() => { try { @@ -86,6 +88,8 @@ const Dashboard = ({ user, onLogout }) => { return ; case 'models': return ; + case 'agents': + return ; case 'calibration': return ; case 'studies': diff --git a/src/website/src/utils/api.js b/src/website/src/utils/api.js index 3b82ff2..13c20aa 100644 --- a/src/website/src/utils/api.js +++ b/src/website/src/utils/api.js @@ -618,6 +618,33 @@ export const getModelComparison = async (params = {}) => { } }; +const getAgentAnalytics = async (path, fallbackError) => { + try { + const response = await fetch( + `${process.env.REACT_APP_BACKEND_HOST}:${process.env.REACT_APP_BACKEND_PORT}/api/analytics/agents/${path}`, + { + method: "GET", + headers: { "Content-Type": "application/json" }, + credentials: "include", + }, + ); + const body = await response.json(); + if (!response.ok) { + return { ok: false, error: body["detail"] || `${response.status}: ${response.statusText}` }; + } + return { ok: true, data: body }; + } catch (error) { + console.error("Error fetching agent analytics:", error); + return { ok: false, error: fallbackError }; + } +}; + +export const getAgentOverview = (params = {}) => + getAgentAnalytics(`overview?${new URLSearchParams(params)}`, "An unexpected error occurred. Please try again."); + +export const getAgentRunDetail = (taskId) => + getAgentAnalytics(`runs/${encodeURIComponent(taskId)}`, "Failed to load agent run detail"); + /** * Get user engagement data * From a1515e08a74a1b17a4478ffd94056d64869a24f7 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 13:45:27 +0200 Subject: [PATCH 020/115] feat: agent study telemetry metrics --- src/backend/routers/analytics/studies.py | 8 ++++++++ .../src/components/analytics/AgentResults.js | 20 +++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/backend/routers/analytics/studies.py b/src/backend/routers/analytics/studies.py index 348d35d..ca9670f 100644 --- a/src/backend/routers/analytics/studies.py +++ b/src/backend/routers/analytics/studies.py @@ -768,6 +768,7 @@ def evaluate_study_agents( t.study_arm_is_baseline AS is_baseline, COUNT(DISTINCT t.task_id) AS total_tasks, COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'failed') AS failed_tasks, COUNT(DISTINCT COALESCE(t.owner_user_id, t.session_id)) AS total_participants, AVG(t.total_steps) AS avg_steps, SUM(t.input_tokens) AS total_input_tokens, @@ -785,8 +786,11 @@ def evaluate_study_agents( event_query = """ SELECT t.profile_id, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_request') AS model_requests, COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, + COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_request') AS tool_requests, COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, + COUNT(e.event_id) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures, AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call') AS avg_model_latency_ms, SUM(e.total_tokens) AS total_tokens FROM agent_task t @@ -846,12 +850,16 @@ def _i(value) -> int: "total_tasks": total_tasks, "completed_tasks": _i(row.completed_tasks), "completion_rate": _i(row.completed_tasks) / max(total_tasks, 1), + "failed_tasks": _i(row.failed_tasks), "total_participants": _i(row.total_participants), "avg_steps": _f(row.avg_steps), "total_input_tokens": _i(row.total_input_tokens), "total_output_tokens": _i(row.total_output_tokens), + "model_requests": _i(events.model_requests) if events else 0, "model_calls": _i(events.model_calls) if events else 0, + "tool_requests": _i(events.tool_requests) if events else 0, "tool_calls": _i(events.tool_calls) if events else 0, + "failures": _i(events.failures) if events else 0, "avg_model_latency_ms": ( _f(events.avg_model_latency_ms) if events else 0.0 ), diff --git a/src/website/src/components/analytics/AgentResults.js b/src/website/src/components/analytics/AgentResults.js index d56674d..a8548d6 100644 --- a/src/website/src/components/analytics/AgentResults.js +++ b/src/website/src/components/analytics/AgentResults.js @@ -164,6 +164,10 @@ const ArmCard = ({ arm }) => { {m.total_tasks || 0} ({m.completed_tasks || 0} completed)
+
+ Failed tasks: + {m.failed_tasks || 0} +
Completion: @@ -175,12 +179,20 @@ const ArmCard = ({ arm }) => { {m.total_participants || 0}
- Model calls: - {m.model_calls || 0} + Model events: + + {m.model_requests || 0} requested / {m.model_calls || 0} completed + +
+
+ Tool events: + + {m.tool_requests || 0} requested / {m.tool_calls || 0} completed +
- Tool calls: - {m.tool_calls || 0} + Telemetry failures: + {m.failures || 0}
Avg steps: From 63aa3560c495ebda408151b60a59f6ca6bbc7bb5 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 13:46:22 +0200 Subject: [PATCH 021/115] fix: propagate upstream rate limits correctly --- src/agents/inference.py | 101 +++++++++++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/src/agents/inference.py b/src/agents/inference.py index ec4c18f..84f28f7 100644 --- a/src/agents/inference.py +++ b/src/agents/inference.py @@ -24,6 +24,8 @@ import hashlib import json import logging +import math +import re import time import uuid from collections import Counter @@ -63,6 +65,22 @@ _UPSTREAM_TIMEOUT_SECONDS = 120 +def _retry_after_seconds(headers: httpx.Headers, body: bytes) -> Optional[int]: + """Extract a provider rate-limit delay so agents can back off correctly.""" + header_value = headers.get("retry-after") + try: + if header_value: + return max(1, min(120, math.ceil(float(header_value)))) + except ValueError: + pass + + body_text = body.decode("utf-8", errors="replace") + match = re.search(r"try again in\s+([0-9]+(?:\.[0-9]+)?)s", body_text, re.IGNORECASE) + if match: + return max(1, min(120, math.ceil(float(match.group(1))))) + return None + + async def run_inference( *, task_uuid: uuid.UUID, @@ -347,6 +365,11 @@ async def run_inference( if isinstance(msg, dict) and msg.get("role") == "assistant": msg.pop("reasoning_content", None) + # Schema size is structural telemetry. Token count is estimated in the + # analytics layer because providers use different tokenizers. + tool_schema_bytes = len( + json.dumps(openai_body.get("tools", []), default=str).encode("utf-8") + ) body_bytes = json.dumps(openai_body).encode() role_counts = Counter( m.get("role", "unknown") for m in messages if isinstance(m, dict) @@ -386,6 +409,7 @@ async def run_inference( "openai_passthrough": use_openai_passthrough, "meta_request": meta_request, "requested_model": requested_model, + "tool_schema_bytes": tool_schema_bytes, } upstream_url = upstream.endpoint(responses_api=is_responses_api) @@ -438,41 +462,55 @@ def _build_record( if streaming: sse_buffer: list[bytes] = [] - captured_status: list[int] = [] stream_aborted: list[bool] = [False] + stream_client = httpx.AsyncClient(timeout=_UPSTREAM_TIMEOUT_SECONDS) + stream_request = stream_client.build_request( + "POST", + upstream_url, + content=body_bytes, + headers=upstream_headers, + ) + upstream_stream = await stream_client.send(stream_request, stream=True) + + # Do not turn an upstream 429 into a successful-looking SSE response. + # Goose retries when it sees the status and Retry-After; yielding the + # error body through StreamingResponse would otherwise default to 200. + if upstream_stream.status_code >= 400: + error_body = await upstream_stream.aread() + retry_after = _retry_after_seconds(upstream_stream.headers, error_body) + await upstream_stream.aclose() + await stream_client.aclose() + latency_ms = int((time.monotonic() - t0) * 1000) + record = _build_record(None, None, None, None, None, latency_ms, upstream_stream.status_code) + _log_record(record) + if record_observation_events: + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + response_headers = {} + if retry_after is not None: + response_headers["Retry-After"] = str(retry_after) + return Response( + content=error_body, + status_code=upstream_stream.status_code, + headers=response_headers, + media_type=upstream_stream.headers.get("content-type", "application/json"), + ) + async def _stream(): try: - async with httpx.AsyncClient( - timeout=_UPSTREAM_TIMEOUT_SECONDS - ) as client: - async with client.stream( - "POST", - upstream_url, - content=body_bytes, - headers=upstream_headers, - ) as upstream_resp: - captured_status.append(upstream_resp.status_code) - logging.info( - f"[Agent/inference] ← upstream " - f"status={upstream_resp.status_code} (stream)" - ) - if upstream_resp.status_code >= 400: - body = await upstream_resp.aread() - logging.error( - f"[Agent/inference] upstream error body: " - f"{body.decode('utf-8', errors='replace')}" - ) - sse_buffer.append(body) - yield body - return - async for chunk in upstream_resp.aiter_bytes(): - sse_buffer.append(chunk) - yield chunk + logging.info( + f"[Agent/inference] ← upstream " + f"status={upstream_stream.status_code} (stream)" + ) + async for chunk in upstream_stream.aiter_bytes(): + sse_buffer.append(chunk) + yield chunk except Exception as e: stream_aborted[0] = True logging.warning(f"[Agent/inference] stream aborted mid-flight — {e}") finally: + await upstream_stream.aclose() + await stream_client.aclose() # Telemetry is written in `finally` so an aborted stream still # produces a row — a dropped connection is itself a finding. latency_ms = int((time.monotonic() - t0) * 1000) @@ -495,7 +533,7 @@ async def _stream(): finish_reason, response_text, latency_ms, - captured_status[0] if captured_status else 0, + upstream_stream.status_code, ) _log_record(record) if record_observation_events: @@ -544,9 +582,16 @@ async def _stream(): if record_observation_events: write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + response_headers = {} + if upstream_resp.status_code == 429: + retry_after = _retry_after_seconds(upstream_resp.headers, upstream_resp.content) + if retry_after is not None: + response_headers["Retry-After"] = str(retry_after) + return Response( content=upstream_resp.content, status_code=upstream_resp.status_code, + headers=response_headers, media_type="application/json", ) From b994abe64029d6a67ee826b31f004bf325d9185f Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 13:47:11 +0200 Subject: [PATCH 022/115] approval policy forwarding --- src/backend/routers/agents.py | 6 ++++ tests/backend_tests/test_managed_acp.py | 41 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py index 0317e26..729e73a 100644 --- a/src/backend/routers/agents.py +++ b/src/backend/routers/agents.py @@ -121,6 +121,9 @@ def create_agent_task( "framework_version": existing.framework_version, "agent_profile": existing.agent_profile, "model": existing.model, + "agent_launch": { + "approval_policy": existing.approval_policy, + }, }, status_code=200, ) @@ -177,6 +180,9 @@ def create_agent_task( # This is only the runtime's displayed/requested model. The inference relay # still enforces the immutable task snapshot on every model call. "model": task.model, + "agent_launch": { + "approval_policy": task.approval_policy, + }, }, status_code=201, ) diff --git a/tests/backend_tests/test_managed_acp.py b/tests/backend_tests/test_managed_acp.py index 30f5511..c0da9cd 100644 --- a/tests/backend_tests/test_managed_acp.py +++ b/tests/backend_tests/test_managed_acp.py @@ -1,4 +1,5 @@ import asyncio +import json import uuid from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -21,6 +22,7 @@ run_managed_inference, ) from backend.routers.agent.ingest import get_agent_run +from backend.routers.agents import TaskCreateRequest, create_agent_task from backend.routers.agent.profiles import AgentProfilePayload from backend.routers.analytics.auth_utils import AuthenticatedUser @@ -189,6 +191,45 @@ def test_managed_policy_applies_the_assigned_command_allowlist(): assert policy["commands_allowlist"] == ["git", "rg"] +def test_plugin_task_response_includes_the_assigned_approval_policy(): + session_id = uuid.uuid4() + user_id = uuid.uuid4() + profile = SimpleNamespace( + name="goose-per-step", + model="study-model", + approval_policy="per_step", + tools_json="[]", + temperature=None, + framework_version="goose", + profile_id=uuid.uuid4(), + ) + assignment = SimpleNamespace( + profile=profile, + study_id=uuid.uuid4(), + assignment_id=uuid.uuid4(), + arm_name="treatment", + is_baseline=False, + ) + task = SimpleNamespace( + task_id=uuid.uuid4(), + model="study-model", + approval_policy="per_step", + ) + db = MagicMock() + app = MagicMock() + app.get_db_session.return_value = db + + with patch("backend.routers.agents.crud.get_agent_task", return_value=None), patch( + "backend.routers.agents.crud.get_session_by_id", + return_value=SimpleNamespace(user_id=user_id), + ), patch("agents.registry.resolve_assignment_context", return_value=assignment), patch( + "backend.routers.agents.resolve_store_agent_content", return_value=False + ), patch("backend.routers.agents.crud.create_agent_task", return_value=task): + response = create_agent_task(TaskCreateRequest(task_id=task.task_id), app, session_id) + + assert json.loads(response.body)["agent_launch"] == {"approval_policy": "per_step"} + + def test_profile_validation_rejects_tools_from_another_runtime(): with pytest.raises(ValueError, match="unsupported by code4me2-agent"): AgentProfilePayload( From c35a75f20ab0e9c1ada52499d7bbce22bcb125d4 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 14 Sep 2026 13:47:45 +0200 Subject: [PATCH 023/115] keep view hash in sync --- src/website/src/pages/Dashboard.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/website/src/pages/Dashboard.js b/src/website/src/pages/Dashboard.js index 52bf442..a1b7bcc 100644 --- a/src/website/src/pages/Dashboard.js +++ b/src/website/src/pages/Dashboard.js @@ -64,13 +64,9 @@ const Dashboard = ({ user, onLogout }) => { if (timeWindow) { url.searchParams.set('timeWindow', timeWindow); } - // Preserve hash only if not being used for the view shortcut - if (window.location.hash && window.location.hash.slice(1) !== activeView) { - // keep existing hash - } else { - // set hash to current view for quick copy - url.hash = '#' + activeView; - } + // Keep the hash aligned with the active view so copied dashboard links + // reopen the same panel instead of a stale previous view. + url.hash = '#' + activeView; window.history.replaceState({}, '', url.toString()); } catch (_) { // ignore URL update errors From 9567cb38b6274ef0497ab66231f0b1e4ac0a9918 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Fri, 18 Sep 2026 12:17:38 +0200 Subject: [PATCH 024/115] for test --- .dockerignore | 4 + .gitignore | 1 + README.md | 111 + docker-compose.dev-arm.yaml | 2 +- pytest.ini | 1 + requirements.txt | 9 + .../dev/generate_canonical_event_schema.py | 41 + scripts/dev/seed_local_dev.sh | 53 + scripts/dev/seed_research_study.py | 1532 ++++++++ session_id | 0 src/agents/event_writer.py | 135 +- src/agents/inference.py | 30 +- src/agents/ingest.py | 66 +- src/agents/provider.py | 249 +- src/agents/registry.py | 257 +- src/backend/completion/__init__.py | 43 +- src/backend/routers/__init__.py | 5 + src/backend/routers/acp/__init__.py | 221 +- src/backend/routers/agent/consent.py | 34 +- src/backend/routers/agent/ingest.py | 24 +- src/backend/routers/agent/profiles.py | 477 +-- src/backend/routers/agents.py | 141 +- src/backend/routers/analytics/agents.py | 437 +-- src/backend/routers/analytics/auth_utils.py | 17 +- src/backend/routers/analytics/studies.py | 200 +- src/backend/routers/research/__init__.py | 91 + src/backend/routers/research/access.py | 276 ++ src/backend/routers/research/agents.py | 538 +++ src/backend/routers/research/bootstrap.py | 582 +++ src/backend/routers/research/join.py | 195 + src/backend/routers/research/operations.py | 122 + src/backend/routers/research/packages.py | 202 + src/backend/routers/research/participants.py | 279 ++ src/backend/routers/research/providers.py | 238 ++ src/backend/routers/research/read_models.py | 112 + src/backend/routers/research/researchers.py | 76 + src/backend/routers/research/sessions.py | 486 +++ src/backend/routers/research/studies.py | 894 +++++ src/backend/routers/research/telemetry.py | 217 ++ src/database/crud.py | 598 +-- src/database/db_schemas.py | 325 +- src/database/migration/env.py | 10 +- .../8a0084080b46_consolidated_schema.py | 554 +++ ...9cc1bc_baseline_migration_from_init_sql.py | 27 - .../versions/d1a2b3c4e5f7_add_agent_tables.py | 396 -- ...2b3c4d5f6a8_seed_default_agent_profiles.py | 120 - ...6a7b9_add_managed_agent_policy_snapshot.py | 27 - ...f7a9_add_experiment_execution_integrity.py | 328 -- src/database/research_schemas.py | 549 +++ src/main.py | 11 +- src/research/__init__.py | 8 + src/research/analysis/__init__.py | 11 + src/research/analysis/operations/__init__.py | 99 + src/research/analysis/operations/enums.py | 118 + src/research/analysis/operations/health.py | 271 ++ .../analysis/operations/kill_switch.py | 204 ++ src/research/analysis/operations/models.py | 276 ++ .../analysis/operations/release_gate.py | 185 + src/research/analysis/operations/retention.py | 180 + src/research/analysis/operations/store.py | 382 ++ src/research/analysis/read_models/__init__.py | 78 + .../analysis/read_models/dashboard.py | 541 +++ src/research/analysis/read_models/enums.py | 38 + src/research/analysis/read_models/models.py | 141 + src/research/analysis/read_models/rbac.py | 128 + src/research/analysis/read_models/service.py | 244 ++ src/research/analysis/read_models/store.py | 144 + src/research/canonical.py | 55 + src/research/compatibility/__init__.py | 20 + src/research/compatibility/canonical.py | 41 + src/research/compatibility/enums.py | 99 + src/research/compatibility/evaluate.py | 325 ++ src/research/compatibility/models.py | 169 + src/research/compatibility/receipt.py | 444 +++ src/research/compatibility/redaction.py | 126 + src/research/compatibility/store.py | 111 + src/research/participants/__init__.py | 14 + src/research/participants/enums.py | 111 + src/research/participants/identity.py | 937 +++++ src/research/participants/models.py | 284 ++ src/research/participants/retention.py | 923 +++++ src/research/runtime/__init__.py | 13 + src/research/runtime/assignment/__init__.py | 50 + src/research/runtime/assignment/enums.py | 77 + src/research/runtime/assignment/exposure.py | 120 + src/research/runtime/assignment/models.py | 126 + src/research/runtime/assignment/service.py | 220 ++ src/research/runtime/assignment/store.py | 170 + src/research/runtime/bootstrap/__init__.py | 95 + src/research/runtime/bootstrap/capability.py | 204 ++ src/research/runtime/bootstrap/models.py | 297 ++ src/research/runtime/bootstrap/service.py | 513 +++ src/research/runtime/bootstrap/signer.py | 113 + src/research/runtime/bootstrap/store.py | 28 + src/research/runtime/sessions/__init__.py | 78 + src/research/runtime/sessions/enums.py | 74 + src/research/runtime/sessions/models.py | 173 + src/research/runtime/sessions/service.py | 559 +++ src/research/runtime/sessions/store.py | 316 ++ src/research/study/__init__.py | 13 + src/research/study/agents/__init__.py | 84 + src/research/study/agents/distributions.py | 224 ++ src/research/study/agents/enums.py | 116 + src/research/study/agents/manifest_import.py | 403 ++ src/research/study/agents/models.py | 282 ++ src/research/study/agents/registry.py | 784 ++++ src/research/study/agents/resolver.py | 172 + src/research/study/agents/store.py | 205 ++ src/research/study/lifecycle.py | 111 + src/research/study/packaging/__init__.py | 93 + src/research/study/packaging/conformance.py | 347 ++ src/research/study/packaging/enums.py | 77 + src/research/study/packaging/models.py | 352 ++ src/research/study/packaging/packager.py | 97 + src/research/study/packaging/resolver.py | 95 + src/research/study/packaging/store.py | 339 ++ src/research/study/packaging/verifier.py | 340 ++ src/research/study/protocol/__init__.py | 122 + src/research/study/protocol/canonical.py | 159 + src/research/study/protocol/enums.py | 179 + src/research/study/protocol/join_code.py | 74 + src/research/study/protocol/models.py | 318 ++ src/research/study/protocol/publication.py | 283 ++ src/research/study/protocol/store.py | 526 +++ src/research/study/protocol/validation.py | 990 +++++ src/research/telemetry/__init__.py | 73 + src/research/telemetry/adapters.py | 153 + src/research/telemetry/builder.py | 184 + src/research/telemetry/enums.py | 146 + src/research/telemetry/ingestion/__init__.py | 66 + src/research/telemetry/ingestion/enums.py | 101 + src/research/telemetry/ingestion/errors.py | 48 + src/research/telemetry/ingestion/models.py | 216 ++ src/research/telemetry/ingestion/service.py | 738 ++++ src/research/telemetry/ingestion/store.py | 373 ++ .../telemetry/ingestion/validation.py | 137 + src/research/telemetry/models.py | 146 + .../telemetry/normalization/__init__.py | 81 + .../telemetry/normalization/generic_acp.py | 653 ++++ .../telemetry/normalization/models.py | 47 + src/research/telemetry/privacy/__init__.py | 50 + src/research/telemetry/privacy/classify.py | 250 ++ src/research/telemetry/privacy/engine.py | 307 ++ .../telemetry/projections/__init__.py | 38 + src/research/telemetry/projections/models.py | 83 + src/research/telemetry/projections/service.py | 197 + src/research/telemetry/schema/__init__.py | 90 + .../telemetry/schema/canonical_event_v1.json | 503 +++ src/research/telemetry/validation.py | 239 ++ src/response_models.py | 6 + src/website/src/App.js | 172 +- src/website/src/__tests__/App.routing.test.js | 63 + .../components/analytics/AgentAnalytics.css | 2 + .../components/analytics/AgentAnalytics.js | 22 +- .../src/components/analytics/AgentResults.js | 14 +- .../analytics/AnalyticsNavigation.js | 56 +- .../components/analytics/StudyManagement.js | 13 +- src/website/src/pages/AgentProfiles.css | 61 + src/website/src/pages/AgentProfiles.js | 429 ++- src/website/src/pages/Dashboard.js | 8 +- .../src/pages/__tests__/AgentProfiles.test.js | 218 ++ .../pages/__tests__/AgentProfiles422.test.js | 147 + .../src/pages/research/ResearchEnrollment.css | 56 + .../src/pages/research/ResearchEnrollment.js | 548 +++ .../src/pages/research/ResearchJoin.js | 301 ++ .../src/pages/research/ResearchStudies.css | 57 + .../src/pages/research/ResearchStudies.js | 774 ++++ .../pages/research/ResearchStudyEditor.css | 114 + .../src/pages/research/ResearchStudyEditor.js | 1155 ++++++ .../__tests__/ResearchEnrollment.test.js | 255 ++ .../research/__tests__/ResearchJoin.test.js | 189 + .../__tests__/ResearchStudies.test.js | 229 ++ .../__tests__/ResearchStudyEditor.test.js | 249 ++ .../__tests__/ResearchStudyEditor422.test.js | 105 + .../pages/research/__tests__/protocol.test.js | 234 ++ src/website/src/pages/research/protocol.js | 371 ++ src/website/src/pages/research/research.css | 381 ++ .../__tests__/researchApi.contract.test.js | 77 + src/website/src/utils/api.js | 522 ++- tests/backend_tests/research/__init__.py | 1 + tests/backend_tests/research/_canary.py | 66 + .../research/_retention_fakes.py | 158 + tests/backend_tests/research/test_analysis.py | 400 ++ .../research/test_analytics_canonical.py | 182 + .../research/test_compatibility.py | 619 ++++ .../research/test_consolidation_wave1.py | 1021 ++++++ .../research/test_distributions.py | 213 ++ .../research/test_integration.py | 3243 +++++++++++++++++ .../research/test_legacy_adapter.py | 93 + .../research/test_manifest_import.py | 235 ++ .../backend_tests/research/test_onboarding.py | 477 +++ .../test_phase03_enrollment_sessions.py | 391 ++ .../research/test_phase04_approvals.py | 157 + tests/backend_tests/research/test_runtime.py | 1783 +++++++++ tests/backend_tests/research/test_study.py | 2513 +++++++++++++ .../backend_tests/research/test_telemetry.py | 2526 +++++++++++++ .../test_study_evaluation_acceptance.py | 83 + tests/backend_tests/routers/user/test_get.py | 5 + .../test_TemplateCompletionModel.py | 3 +- .../test_chat_completion_model.py | 3 + tests/backend_tests/test_managed_acp.py | 148 +- tests/database_tests/test_crud.py | 5 +- tests/database_tests/test_migration_graph.py | 49 + .../test_phase03_concurrency.py | 411 +++ .../test_phase05_envelope_roundtrip.py | 172 + .../test_phase06_dashboard_canonical.py | 144 + .../test_research_schema_consolidation.py | 195 + .../test_research_schema_lifecycle.py | 308 ++ tests/database_tests/test_wave1_live_study.py | 223 ++ .../research/agents/release_codex_acp_v1.json | 49 + .../agents/snapshot_declared_unobserved.json | 70 + .../bootstrap/incompatible_environment.json | 15 + .../research/bootstrap/manifest_valid.json | 73 + .../host_intellij_2026_2__agent_codex_v1.json | 192 + ...intellij_2026_2__partial_tool_updates.json | 176 + .../parse_failure_transcript.json | 25 + tests/fixtures/research/exports/canaries.json | 9 + .../ingestion/batch_duplicate_conflict.json | 143 + .../research/ingestion/batch_valid.json | 143 + .../ingestion/mixed_valid_revoked.json | 143 + .../research/packaging/conformance_cases.json | 61 + .../packaging/digest_mismatch_manifest.json | 53 + .../packaging/unsafe_path_manifest.json | 53 + .../research/packaging/valid_manifest.json | 53 + .../protocol/approved_study_protocol_v1.json | 119 + .../acp_completeness_transcript.json | 113 + .../telemetry/acp_transcript_excerpt.json | 93 + .../telemetry/canonical_event_accepted.json | 74 + .../telemetry/golden_acp_candidates.json | 58 + .../telemetry/golden_acp_completeness.json | 182 + .../research/telemetry/secrets_canary.json | 23 + uv.lock | 1143 ++++++ 232 files changed, 56091 insertions(+), 2607 deletions(-) create mode 100644 scripts/dev/generate_canonical_event_schema.py create mode 100755 scripts/dev/seed_local_dev.sh create mode 100644 scripts/dev/seed_research_study.py create mode 100644 session_id create mode 100644 src/backend/routers/research/__init__.py create mode 100644 src/backend/routers/research/access.py create mode 100644 src/backend/routers/research/agents.py create mode 100644 src/backend/routers/research/bootstrap.py create mode 100644 src/backend/routers/research/join.py create mode 100644 src/backend/routers/research/operations.py create mode 100644 src/backend/routers/research/packages.py create mode 100644 src/backend/routers/research/participants.py create mode 100644 src/backend/routers/research/providers.py create mode 100644 src/backend/routers/research/read_models.py create mode 100644 src/backend/routers/research/researchers.py create mode 100644 src/backend/routers/research/sessions.py create mode 100644 src/backend/routers/research/studies.py create mode 100644 src/backend/routers/research/telemetry.py create mode 100644 src/database/migration/versions/8a0084080b46_consolidated_schema.py delete mode 100644 src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py delete mode 100644 src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py delete mode 100644 src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py delete mode 100644 src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py delete mode 100644 src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py create mode 100644 src/database/research_schemas.py create mode 100644 src/research/__init__.py create mode 100644 src/research/analysis/__init__.py create mode 100644 src/research/analysis/operations/__init__.py create mode 100644 src/research/analysis/operations/enums.py create mode 100644 src/research/analysis/operations/health.py create mode 100644 src/research/analysis/operations/kill_switch.py create mode 100644 src/research/analysis/operations/models.py create mode 100644 src/research/analysis/operations/release_gate.py create mode 100644 src/research/analysis/operations/retention.py create mode 100644 src/research/analysis/operations/store.py create mode 100644 src/research/analysis/read_models/__init__.py create mode 100644 src/research/analysis/read_models/dashboard.py create mode 100644 src/research/analysis/read_models/enums.py create mode 100644 src/research/analysis/read_models/models.py create mode 100644 src/research/analysis/read_models/rbac.py create mode 100644 src/research/analysis/read_models/service.py create mode 100644 src/research/analysis/read_models/store.py create mode 100644 src/research/canonical.py create mode 100644 src/research/compatibility/__init__.py create mode 100644 src/research/compatibility/canonical.py create mode 100644 src/research/compatibility/enums.py create mode 100644 src/research/compatibility/evaluate.py create mode 100644 src/research/compatibility/models.py create mode 100644 src/research/compatibility/receipt.py create mode 100644 src/research/compatibility/redaction.py create mode 100644 src/research/compatibility/store.py create mode 100644 src/research/participants/__init__.py create mode 100644 src/research/participants/enums.py create mode 100644 src/research/participants/identity.py create mode 100644 src/research/participants/models.py create mode 100644 src/research/participants/retention.py create mode 100644 src/research/runtime/__init__.py create mode 100644 src/research/runtime/assignment/__init__.py create mode 100644 src/research/runtime/assignment/enums.py create mode 100644 src/research/runtime/assignment/exposure.py create mode 100644 src/research/runtime/assignment/models.py create mode 100644 src/research/runtime/assignment/service.py create mode 100644 src/research/runtime/assignment/store.py create mode 100644 src/research/runtime/bootstrap/__init__.py create mode 100644 src/research/runtime/bootstrap/capability.py create mode 100644 src/research/runtime/bootstrap/models.py create mode 100644 src/research/runtime/bootstrap/service.py create mode 100644 src/research/runtime/bootstrap/signer.py create mode 100644 src/research/runtime/bootstrap/store.py create mode 100644 src/research/runtime/sessions/__init__.py create mode 100644 src/research/runtime/sessions/enums.py create mode 100644 src/research/runtime/sessions/models.py create mode 100644 src/research/runtime/sessions/service.py create mode 100644 src/research/runtime/sessions/store.py create mode 100644 src/research/study/__init__.py create mode 100644 src/research/study/agents/__init__.py create mode 100644 src/research/study/agents/distributions.py create mode 100644 src/research/study/agents/enums.py create mode 100644 src/research/study/agents/manifest_import.py create mode 100644 src/research/study/agents/models.py create mode 100644 src/research/study/agents/registry.py create mode 100644 src/research/study/agents/resolver.py create mode 100644 src/research/study/agents/store.py create mode 100644 src/research/study/lifecycle.py create mode 100644 src/research/study/packaging/__init__.py create mode 100644 src/research/study/packaging/conformance.py create mode 100644 src/research/study/packaging/enums.py create mode 100644 src/research/study/packaging/models.py create mode 100644 src/research/study/packaging/packager.py create mode 100644 src/research/study/packaging/resolver.py create mode 100644 src/research/study/packaging/store.py create mode 100644 src/research/study/packaging/verifier.py create mode 100644 src/research/study/protocol/__init__.py create mode 100644 src/research/study/protocol/canonical.py create mode 100644 src/research/study/protocol/enums.py create mode 100644 src/research/study/protocol/join_code.py create mode 100644 src/research/study/protocol/models.py create mode 100644 src/research/study/protocol/publication.py create mode 100644 src/research/study/protocol/store.py create mode 100644 src/research/study/protocol/validation.py create mode 100644 src/research/telemetry/__init__.py create mode 100644 src/research/telemetry/adapters.py create mode 100644 src/research/telemetry/builder.py create mode 100644 src/research/telemetry/enums.py create mode 100644 src/research/telemetry/ingestion/__init__.py create mode 100644 src/research/telemetry/ingestion/enums.py create mode 100644 src/research/telemetry/ingestion/errors.py create mode 100644 src/research/telemetry/ingestion/models.py create mode 100644 src/research/telemetry/ingestion/service.py create mode 100644 src/research/telemetry/ingestion/store.py create mode 100644 src/research/telemetry/ingestion/validation.py create mode 100644 src/research/telemetry/models.py create mode 100644 src/research/telemetry/normalization/__init__.py create mode 100644 src/research/telemetry/normalization/generic_acp.py create mode 100644 src/research/telemetry/normalization/models.py create mode 100644 src/research/telemetry/privacy/__init__.py create mode 100644 src/research/telemetry/privacy/classify.py create mode 100644 src/research/telemetry/privacy/engine.py create mode 100644 src/research/telemetry/projections/__init__.py create mode 100644 src/research/telemetry/projections/models.py create mode 100644 src/research/telemetry/projections/service.py create mode 100644 src/research/telemetry/schema/__init__.py create mode 100644 src/research/telemetry/schema/canonical_event_v1.json create mode 100644 src/research/telemetry/validation.py create mode 100644 src/website/src/__tests__/App.routing.test.js create mode 100644 src/website/src/pages/__tests__/AgentProfiles.test.js create mode 100644 src/website/src/pages/__tests__/AgentProfiles422.test.js create mode 100644 src/website/src/pages/research/ResearchEnrollment.css create mode 100644 src/website/src/pages/research/ResearchEnrollment.js create mode 100644 src/website/src/pages/research/ResearchJoin.js create mode 100644 src/website/src/pages/research/ResearchStudies.css create mode 100644 src/website/src/pages/research/ResearchStudies.js create mode 100644 src/website/src/pages/research/ResearchStudyEditor.css create mode 100644 src/website/src/pages/research/ResearchStudyEditor.js create mode 100644 src/website/src/pages/research/__tests__/ResearchEnrollment.test.js create mode 100644 src/website/src/pages/research/__tests__/ResearchJoin.test.js create mode 100644 src/website/src/pages/research/__tests__/ResearchStudies.test.js create mode 100644 src/website/src/pages/research/__tests__/ResearchStudyEditor.test.js create mode 100644 src/website/src/pages/research/__tests__/ResearchStudyEditor422.test.js create mode 100644 src/website/src/pages/research/__tests__/protocol.test.js create mode 100644 src/website/src/pages/research/protocol.js create mode 100644 src/website/src/pages/research/research.css create mode 100644 src/website/src/utils/__tests__/researchApi.contract.test.js create mode 100644 tests/backend_tests/research/__init__.py create mode 100644 tests/backend_tests/research/_canary.py create mode 100644 tests/backend_tests/research/_retention_fakes.py create mode 100644 tests/backend_tests/research/test_analysis.py create mode 100644 tests/backend_tests/research/test_analytics_canonical.py create mode 100644 tests/backend_tests/research/test_compatibility.py create mode 100644 tests/backend_tests/research/test_consolidation_wave1.py create mode 100644 tests/backend_tests/research/test_distributions.py create mode 100644 tests/backend_tests/research/test_integration.py create mode 100644 tests/backend_tests/research/test_legacy_adapter.py create mode 100644 tests/backend_tests/research/test_manifest_import.py create mode 100644 tests/backend_tests/research/test_onboarding.py create mode 100644 tests/backend_tests/research/test_phase03_enrollment_sessions.py create mode 100644 tests/backend_tests/research/test_phase04_approvals.py create mode 100644 tests/backend_tests/research/test_runtime.py create mode 100644 tests/backend_tests/research/test_study.py create mode 100644 tests/backend_tests/research/test_telemetry.py create mode 100644 tests/backend_tests/routers/test_study_evaluation_acceptance.py create mode 100644 tests/database_tests/test_migration_graph.py create mode 100644 tests/database_tests/test_phase03_concurrency.py create mode 100644 tests/database_tests/test_phase05_envelope_roundtrip.py create mode 100644 tests/database_tests/test_phase06_dashboard_canonical.py create mode 100644 tests/database_tests/test_research_schema_consolidation.py create mode 100644 tests/database_tests/test_research_schema_lifecycle.py create mode 100644 tests/database_tests/test_wave1_live_study.py create mode 100644 tests/fixtures/research/agents/release_codex_acp_v1.json create mode 100644 tests/fixtures/research/agents/snapshot_declared_unobserved.json create mode 100644 tests/fixtures/research/bootstrap/incompatible_environment.json create mode 100644 tests/fixtures/research/bootstrap/manifest_valid.json create mode 100644 tests/fixtures/research/compatibility/host_intellij_2026_2__agent_codex_v1.json create mode 100644 tests/fixtures/research/compatibility/host_intellij_2026_2__partial_tool_updates.json create mode 100644 tests/fixtures/research/compatibility/parse_failure_transcript.json create mode 100644 tests/fixtures/research/exports/canaries.json create mode 100644 tests/fixtures/research/ingestion/batch_duplicate_conflict.json create mode 100644 tests/fixtures/research/ingestion/batch_valid.json create mode 100644 tests/fixtures/research/ingestion/mixed_valid_revoked.json create mode 100644 tests/fixtures/research/packaging/conformance_cases.json create mode 100644 tests/fixtures/research/packaging/digest_mismatch_manifest.json create mode 100644 tests/fixtures/research/packaging/unsafe_path_manifest.json create mode 100644 tests/fixtures/research/packaging/valid_manifest.json create mode 100644 tests/fixtures/research/protocol/approved_study_protocol_v1.json create mode 100644 tests/fixtures/research/telemetry/acp_completeness_transcript.json create mode 100644 tests/fixtures/research/telemetry/acp_transcript_excerpt.json create mode 100644 tests/fixtures/research/telemetry/canonical_event_accepted.json create mode 100644 tests/fixtures/research/telemetry/golden_acp_candidates.json create mode 100644 tests/fixtures/research/telemetry/golden_acp_completeness.json create mode 100644 tests/fixtures/research/telemetry/secrets_canary.json create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore index b60f8e2..4e95503 100644 --- a/.dockerignore +++ b/.dockerignore @@ -31,5 +31,9 @@ data/ Dockerfile* docker-compose*.yml +# Development-only scripts (e.g. the guarded research-study seeder) must never +# be baked into the server image. +scripts/dev/ + # OS .DS_Store diff --git a/.gitignore b/.gitignore index 1fc221c..95aad67 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ dist/ .ruff_cache/ .mypy_cache/ htmlcov/ +*.md \ No newline at end of file diff --git a/README.md b/README.md index 82f6202..5dd21e8 100644 --- a/README.md +++ b/README.md @@ -567,11 +567,122 @@ EMAIL_PASSWORD=your_app_password EMAIL_FROM=noreply@code4me.com ``` +## 🧪 Research Platform + +The research platform (Issue 01+) is mounted under `/api/research` and is +documented in [`docs/research-platform/`](docs/research-platform/). It is +admin/operator-facing: compatibility, protocol publication, registry, bootstrap, +and exports record evidence or define experimental intent and are never exposed +to participants. + +```http +# ACP host capability receipts + compatibility gate (admin) +GET/POST /api/research/compatibility/... + +# Immutable StudyProtocolV1 authoring + publication (admin) +GET/POST /api/research/studies/... + +# Agent registry, artifacts, and capability snapshots (admin) +GET/POST /api/research/agents/... + +# Participant identity, consent, and withdrawal +GET/POST /api/research/participants/... + +# Bootstrap manifest retrieval/validation +GET/POST /api/research/bootstrap/... + +# Research session lifecycle (session capability auth) +GET/POST /api/research/sessions/... + +# Canonical telemetry batch ingestion +GET/POST /api/research/telemetry/... + +# Researcher read models + pilot operations/release gate (RBAC/admin) +GET/POST /api/research/operations/... +``` + +### Seed a fresh database from the built runtime + +A fresh database is made immediately usable by importing the runtime **build +manifest** (never a hand-typed digest), recording the conformance receipt that +qualifies its release, pinning `default-code4me2-agent` to it, marking the +participant-installed profiles (`default-goose`/`default-codex`) as BYOA, and +publishing one study revision with a working `session_policy`. + +```bash +cd code4me2-server +scripts/dev/seed_local_dev.sh +``` + +`seed_local_dev.sh` is a thin driver: it stages the manifest (and any built +archives next to it) into the running backend container and then invokes the +canonical seeder, `scripts/dev/seed_research_study.py --fresh-db`. When an +archive is present its real `sha256`/size are computed and checked against the +manifest, so no digest or size is ever typed by hand. Override the manifest with +`MANIFEST=/path/to/manifest.json`; when an archive is not on disk, supply its +size explicitly with `--artifact-size-override NAME=SIZE` (never guessed). + +The same registration is exposed to CI as an admin JSON endpoint: + +```bash +curl -X POST http://localhost:8008/api/research/agents/releases/import \ + -H "Content-Type: application/json" -H "Cookie: auth_token=$TOKEN" \ + -d '{"manifest": , "artifact_root": "/srv/runtime-staging"}' +``` + +It is idempotent: re-importing the same manifest returns the existing release +(`created: false`) instead of a duplicate. + +### Seed a synthetic study (participant enrollment) + +The participant runbook needs a published study and an ACTIVE enrollment before +the plugin can join. `scripts/dev/seed_research_study.py` mints that onboarding +state by calling the real research services (protocol publication, agent +registry, identity enrollment/consent) — it does not re-implement any domain +logic. It is a development-only tool and refuses to run unless +`CODE4ME_DEV_SEED=1` (or `TEST_MODE=true`) is set. + +```bash +cd code4me2-server +CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_research_study.py \ + --account-email participant@example.com \ + --account-password 'Password123' --create-account +``` + +Common options: `--study-name`, `--agent-id`, `--release-id`, +`--artifact-digest sha256:<64 hex>`, `--os `, `--arch `. Without +`--artifact-digest`/`--os`/`--arch` a deterministic synthetic release for the +current host is used. + +The command prints a copy-pasteable summary. The `enrollment_id` is the opaque +"enrollment code" the participant pastes into IntelliJ's **Tools → Join Research +Study...** action: + +```text + enrollment_id (join code): + enrollment_status: ACTIVE + study_id: + revision_id: + agent_id: code4me-synthetic-agent + release_id: code4me-synthetic-agent-synthetic-1 + artifact_digest: sha256:<64 hex> + login email: participant@example.com + login password: Password123 +``` + +The script is deterministic and safe to re-run: the same arguments resolve the +same account, release, study, published revision and enrollment instead of +piling up duplicates. A password is only printed when this run created the +account (`--create-account`); an existing account is left untouched. The release +is registered `QUALIFIED` and immutable — changing `--artifact-digest` without a +new `--release-id` is rejected rather than mutating a published release. + ## 📖 Documentation - **API Documentation**: http://localhost:8008/docs (when running) - **Database Schema**: See `src/database/resources/documentation/` - **Authentication Workflows**: See `src/backend/resources/documentation/` +- **Research Platform**: See [`docs/research-platform/`](docs/research-platform/) ## 🔒 Production Notes - Restrict CORS origins and cookies in production. diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml index 63c037d..98a1149 100644 --- a/docker-compose.dev-arm.yaml +++ b/docker-compose.dev-arm.yaml @@ -124,7 +124,7 @@ services: ] start_period: 30s interval: 60s - timeout: 10s + timeout: 20s retries: 3 backend: diff --git a/pytest.ini b/pytest.ini index 59a7237..d3526fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,5 @@ pythonpath= src testpaths = tests env = TEST_MODE=true + BOOTSTRAP_SIGNING_SECRET=test-bootstrap-signing-secret ;addopts = --ignore=src/database/migration/versions \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 60d6ec5..8167a48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,9 @@ pytest-mock~=3.10.0 pytest-postgresql~=6.0.0 pytest-env~=1.1.0 hypothesis~=6.70.0 +# Used by tests/backend_tests/research/test_telemetry.py to validate the emitted +# telemetry JSON Schema (Draft 2020-12) directly. +jsonschema~=4.26.0 # ----------------------- # Core Backend @@ -73,3 +76,9 @@ huggingface_hub[cli] # Utilities # ----------------------- requests~=2.32.3 + +# ----------------------- +# Research exports (Issue 12) +# ----------------------- +# Canonical research export artifact writer/reader. The export service degrades +# to CSV/JSONL with a typed EXPORT_WRITER_UNAVAILABLE when this is absent. diff --git a/scripts/dev/generate_canonical_event_schema.py b/scripts/dev/generate_canonical_event_schema.py new file mode 100644 index 0000000..f7e6e9e --- /dev/null +++ b/scripts/dev/generate_canonical_event_schema.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Regenerate the canonical telemetry JSON Schema from the Python model. + +``CanonicalEventV1`` (``research.telemetry.models``) is the single source of +truth for the event envelope; the closed vocabularies come from +``research.telemetry.enums``. This script writes the language-neutral contract +consumed by generated clients. + +It is intentionally dependency-free and side-effect limited to the schema file. +The test suite asserts that the checked-in file equals the generated output, so a +model change that is not accompanied by a regeneration fails fast. + +Usage (from ``code4me2-server/``):: + + python scripts/dev/generate_canonical_event_schema.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SRC = PROJECT_ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from research.telemetry.schema import ( # noqa: E402 + CANONICAL_EVENT_V1_SCHEMA_PATH, + write_canonical_event_v1_schema, +) + + +def main() -> int: + path = write_canonical_event_v1_schema(CANONICAL_EVENT_V1_SCHEMA_PATH) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dev/seed_local_dev.sh b/scripts/dev/seed_local_dev.sh new file mode 100755 index 0000000..ceb0c00 --- /dev/null +++ b/scripts/dev/seed_local_dev.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# One-command local-dev seed for a fresh database. +# +# It stages the *built* runtime manifest (and any archives that exist next to +# it) into the running backend container, then runs the canonical seeder +# (scripts/dev/seed_research_study.py --fresh-db). The seeder imports the +# release, records the conformance receipt that qualifies it, pins +# default-code4me2-agent to it, turns Goose/Codex into BYOA identities, and +# publishes one study revision with a working session policy. +# +# No digest or size is ever typed by hand: when an archive is present its real +# sha256/size are computed inside the container and checked against the manifest. +# +# Usage (from code4me2-server/): +# scripts/dev/seed_local_dev.sh +# MANIFEST=/path/to/manifest.json scripts/dev/seed_local_dev.sh +# scripts/dev/seed_local_dev.sh --builtin-study-name "My Study" +# +# Environment overrides: +# CONTAINER backend container name (default: backend) +# MANIFEST path to the build manifest JSON (default: ../code4me2/src/main/resources/code4me-runtime/manifest.json) +# STAGING in-container staging directory (default: /tmp/code4me-runtime-staging) + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CONTAINER="${CONTAINER:-backend}" +MANIFEST="${MANIFEST:-$(cd "$REPO_ROOT/.." && pwd)/code4me2/src/main/resources/code4me-runtime/manifest.json}" +STAGING="${STAGING:-/tmp/code4me-runtime-staging}" + +if [[ ! -f "$MANIFEST" ]]; then + echo "manifest not found: $MANIFEST" >&2 + echo "set MANIFEST=/path/to/manifest.json (or build the runtime first)" >&2 + exit 1 +fi + +MANIFEST_DIR="$(cd "$(dirname "$MANIFEST")" && pwd)" +MANIFEST_NAME="$(basename "$MANIFEST")" + +echo "==> staging build manifest into container '$CONTAINER'" +docker exec "$CONTAINER" bash -lc "rm -rf '$STAGING' && mkdir -p '$STAGING/code4me-runtime'" +docker cp "$MANIFEST" "$CONTAINER:$STAGING/code4me-runtime/$MANIFEST_NAME" + +shopt -s nullglob +for archive in "$MANIFEST_DIR"/code4me-agent-*.zip; do + echo " + $(basename "$archive")" + docker cp "$archive" "$CONTAINER:$STAGING/code4me-runtime/$(basename "$archive")" +done +shopt -u nullglob + +echo "==> seeding fresh database (import manifest -> qualify -> pin -> publish)" +docker exec -i "$CONTAINER" bash -lc "source activate myenv && cd /app && CODE4ME_DEV_SEED=1 python scripts/dev/seed_research_study.py --fresh-db --manifest '$STAGING/code4me-runtime/$MANIFEST_NAME' --artifact-root '$STAGING' \"\$@\"" -- "$@" diff --git a/scripts/dev/seed_research_study.py b/scripts/dev/seed_research_study.py new file mode 100644 index 0000000..05cfec5 --- /dev/null +++ b/scripts/dev/seed_research_study.py @@ -0,0 +1,1532 @@ +#!/usr/bin/env python3 +"""Seed a synthetic, publishable research study for local onboarding. + +This is the operator tool the participant runbook uses to mint an *enrollment +code*. It performs no domain logic of its own: every state transition delegates +to the real research platform services and stores: + +* ``research.study.protocol.store`` (``create_study`` / ``create_draft`` / + ``persist_revision``) and ``research.study.protocol.publication.publish_revision``; +* ``research.study.agents.store`` (``upsert_release``) with + ``research.study.agents.models.AgentReleaseV1`` (qualification is derived from + conformance evidence, not supplied by the seeder); +* ``research.participants.identity`` (``enroll`` and the + persistence helpers); +* ``database.crud`` for the login account, bound through a + ``research.participants.models.Participant``. + +Everything is digest-pinned and deterministic so the script is safe to re-run: +the same arguments resolve the same account, release, study, published revision +and enrollment instead of piling up duplicates. Nothing secret is printed except +a password the operator explicitly supplied with ``--create-account``. + +Usage (from ``code4me2-server/``, development/test only):: + + CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_research_study.py \\ + --account-email participant@example.com \\ + --account-password 'Password123' --create-account + +``TEST_MODE=true`` is accepted in place of ``CODE4ME_DEV_SEED=1``. Without one of +those guards the script exits before touching a database, so it can never seed a +production deployment by accident. + +The printed ``enrollment_id`` is the opaque "enrollment code" the participant +pastes into the IntelliJ plugin's *Join Research Study...* action. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import sys +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence +from uuid import UUID + +if TYPE_CHECKING: + from research.study.protocol.validation import ValidationError + +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.participants.models import ( + Participant, +) +from research.study.agents.distributions import resolve_distribution_view +from research.study.agents.enums import ( + DistributionMode, + DistributionSourceType, + QualificationStatus, +) +from research.study.agents.manifest_import import build_manifest_release +from research.study.agents.models import ( + AdapterRef, + AgentReleaseV1, + DistributionArtifact, + ReleaseDisplay, +) +from research.study.agents.registry import AgentRegistry +from research.study.packaging import ( + CaseObservation, + ConformanceCaseV1, + ConformanceRunner, +) +from research.study.packaging import store as packaging_store +from research.study.packaging.enums import ConformanceStatus +from research.study.packaging.models import ( + ConformanceCaseResultV1, + ConformanceReceiptV1, + PlatformTriple, +) +from research.study.protocol import store as protocol_store +from research.study.protocol.canonical import protocol_digest +from research.study.protocol.enums import ( + AssignmentStrategy, + CompletionPolicyKind, + PublicationOutcome, + RetentionAction, + RevisionStatus, + ScheduleKind, + TelemetryFieldClass, +) +from research.study.protocol.models import ( + AssignmentPolicy, + CapabilityExpectation, + CompletionPolicy, + EnrollmentPolicy, + EnvironmentRequirements, + FixedSchedule, + PrivacyPolicy, + SessionPolicy, + StudyCondition, + StudyMetadata, + StudyProtocolV1, + TelemetryPolicy, +) +from research.study.protocol.publication import ( + StudyRevision, + lineage_from_revisions, + publish_revision, +) +from research.study.protocol.validation import freeze_protocol_distributions + +# The fixed window is deliberately deterministic (not "now + N days") so the +# canonical protocol digest, and therefore the published revision, is identical +# on every run. ``end_at`` stays far enough in the future to satisfy the +# fixed-schedule expiry check for this synthetic study. +FIXED_SCHEDULE_START = datetime(2024, 1, 1, tzinfo=timezone.utc) +FIXED_SCHEDULE_END = datetime(2099, 1, 1, tzinfo=timezone.utc) + +DEFAULT_STUDY_NAME = "Synthetic Onboarding Study" +DEFAULT_AGENT_ID = "code4me-synthetic-agent" +DEFAULT_RELEASE_VERSION = "1.0.0" +DEFAULT_EXPECTED_PROTOCOL_VERSION = "0.12.1" +DEFAULT_ARTIFACT_SIZE = 1024 + +#: The fresh-DB seeder pins the *built-in* Code4Me runtime distribution and +#: leaves the participant-installed runtimes as BYOA identities. +DEFAULT_BUILTIN_STUDY_NAME = "Code4Me2 Built-in Agent Study" +BUILTIN_DISTRIBUTION_NAME = "default-code4me2-agent" +#: Development-only account created by ``--fresh-db`` seeding. Guarded by the +#: CODE4ME_DEV_SEED environment variable in ``scripts/dev/seed_local_dev.sh`` +#: and by the ``--fresh-db`` flag; never a production credential. +DEFAULT_DEV_OWNER_EMAIL = "research-owner@local.dev" +DEFAULT_DEV_PASSWORD = "Code4me-dev1" +#: ``(profile name, discovery package, command)`` for the BYOA profiles. +BYOA_PROFILE_IDENTITIES = ( + ("default-goose", "goose", "goose"), + ("default-codex", "codex", "codex"), +) + +CONDITIONS = ( + ("control", "Control", 0.5), + ("treatment", "Treatment", 0.5), +) + + +class SeedError(RuntimeError): + """A typed, operator-facing failure (no partial policy is silently kept).""" + + +@dataclass(frozen=True) +class SeedRequest: + """The fully resolved inputs of one seed run.""" + + account_email: str + account_password: Optional[str] + account_name: str + create_account: bool + config_id: Optional[int] + study_name: str + agent_id: str + release_id: str + release_version: str + artifact_digest: str + artifact_path: str + artifact_size: int + os_name: str + arch: str + actor: str + require_capabilities: bool = False + # Distribution mode: PACKAGED (default, digest-pinned artifact) or + # BYOA_EXTERNAL (participant-installed Goose/Codex/...; command/package + # identity instead of an artifact digest). + distribution_mode: str = DistributionMode.PACKAGED.value + agent_command: Optional[str] = None + agent_command_args: tuple[str, ...] = () + agent_package: Optional[str] = None + # Optional link to an existing, server-side AgentProfile (provider/model/ + # tools/policy catalogue). Only the opaque id is pinned into the protocol; + # the profile's provider/base_url/api_key_ref never enter the document. + agent_profile_id: Optional[UUID] = None + + @property + def is_byoa(self) -> bool: + return self.distribution_mode.strip().upper() == DistributionMode.BYOA_EXTERNAL.value + + @property + def study_id(self) -> UUID: + """Deterministic study identity derived from the study name.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/study/{self.study_name}" + ) + + @property + def draft_id(self) -> UUID: + """Deterministic draft identity derived from the study identity.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/draft/{self.study_id}" + ) + + @property + def distribution_id(self) -> UUID: + """Deterministic distribution (AgentProfile) identity. + + The distribution is the single thing a researcher selects; the seed mints + a deterministic one per (study, release, mode) so re-runs are idempotent. + """ + return uuid.uuid5( + uuid.NAMESPACE_URL, + "code4me2://research/distribution/" + f"{self.study_name}/{self.release_id}/{self.distribution_mode}", + ) + + +@dataclass(frozen=True) +class SeedSummary: + """The copy-pasteable result of a seed run.""" + + enrollment_id: str + enrollment_status: str + study_id: str + revision_id: str + agent_id: str + release_id: str + artifact_digest: str + distribution_mode: str + account_email: str + account_password: Optional[str] + account_created: bool + agent_profile_id: Optional[str] = None + distribution_id: Optional[str] = None + + +@dataclass(frozen=True) +class FreshDbRequest: + """The resolved inputs of a "fresh DB immediately usable" seed run. + + It imports the built runtime manifest (never a synthetic digest), records the + conformance evidence that makes the release QUALIFIED, pins the built-in + distribution to that release, marks the participant-installed profiles as + BYOA, and publishes one study revision with a working session policy. + """ + + manifest: dict[str, Any] + artifact_root: Optional[str] = None + artifact_sizes: dict[str, int] = field(default_factory=dict) + study_name: str = DEFAULT_BUILTIN_STUDY_NAME + actor: str = "seed-script" + require_capabilities: bool = False + # Development owner account that owns the seeded profile and study. It is + # created enabled-for-research when missing so a fresh database is usable + # without any per-participant SQL or seed migration. + owner_email: str = DEFAULT_DEV_OWNER_EMAIL + owner_password: str = DEFAULT_DEV_PASSWORD + owner_name: str = "Research Owner" + + @property + def study_id(self) -> UUID: + """Deterministic study identity derived from the study name.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/study/{self.study_name}" + ) + + @property + def draft_id(self) -> UUID: + """Deterministic draft identity derived from the study identity.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/draft/{self.study_id}" + ) + + +@dataclass(frozen=True) +class FreshDbSummary: + """The result of a fresh-DB seed run (non-secret, copy-pasteable).""" + + agent_id: str + release_id: str + release_version: str + qualification: str + distribution_id: str + distribution_verified: bool + supported_platforms: list[dict[str, str]] + study_id: str + revision_id: str + session_policy: dict[str, Any] + skipped_artifacts: list[dict[str, str]] + + +# --------------------------------------------------------------------------- +# Pure planning helpers +# --------------------------------------------------------------------------- + + +def _sha256_hex(*parts: str) -> str: + digest = hashlib.sha256() + for part in parts: + digest.update(part.encode("utf-8")) + return digest.hexdigest() + + +def detect_os() -> str: + """Return the Gradle/plugin platform name for the current host.""" + if sys.platform.startswith("win"): + return "windows" + if sys.platform == "darwin": + return "macos" + return "linux" + + +def detect_arch() -> str: + """Return the Gradle/plugin architecture name for the current host.""" + machine = platform.machine().lower() + if machine in ("aarch64", "arm64"): + return "aarch64" + return "x64" + + +def synthetic_consent_digest(study_name: str, version: str = "1") -> str: + """A stable, non-secret digest standing in for a real consent document.""" + return "sha256:" + _sha256_hex("code4me2://consent", study_name, version) + + +def build_synthetic_release(request: SeedRequest) -> AgentReleaseV1: + """A single-platform synthetic release (no secrets). + + ``PACKAGED`` (the default) pins one digest-pinned distribution artifact; + ``BYOA_EXTERNAL`` pins a participant-installed command/package identity and + publishes no artifact. + """ + byoa = request.is_byoa + identity_digest = ( + (request.agent_command or "") + "|" + (request.agent_package or "") + if byoa + else request.artifact_digest + ) + manifest_digest = "sha256:" + _sha256_hex( + request.agent_id, + request.release_id, + request.release_version, + request.os_name, + request.arch, + request.distribution_mode, + identity_digest, + ) + adapter_digest = "sha256:" + _sha256_hex( + "code4me2://adapter", request.agent_id, request.release_version + ) + return AgentReleaseV1( + agent_id=request.agent_id, + release_id=request.release_id, + version=request.release_version, + display=ReleaseDisplay( + name="Synthetic research agent", + vendor="code4me2", + description="Synthetic release for local study onboarding.", + ), + source_type=DistributionSourceType.RESEARCH_OVERLAY, + source_manifest_digest=manifest_digest, + distribution_mode=( + DistributionMode.BYOA_EXTERNAL if byoa else DistributionMode.PACKAGED + ), + artifacts=( + [] + if byoa + else [ + DistributionArtifact( + os=request.os_name, + arch=request.arch, + path=request.artifact_path, + sha256=request.artifact_digest, + size=request.artifact_size, + ) + ] + ), + agent_command=(request.agent_command or request.agent_package) if byoa else None, + agent_command_args=list(request.agent_command_args) if byoa else [], + agent_package=request.agent_package if byoa else None, + adapter=AdapterRef( + adapter_id=f"{request.agent_id}-adapter", + version="1.0.0", + digest=adapter_digest, + ), + min_protocol_version=DEFAULT_EXPECTED_PROTOCOL_VERSION, + max_protocol_version=DEFAULT_EXPECTED_PROTOCOL_VERSION, + # A release starts UNQUALIFIED; ``ensure_release`` records a synthetic + # passing conformance receipt, which is the only promotion path. + qualification_status=QualificationStatus.UNQUALIFIED, + ) + + +def build_study_protocol( + request: SeedRequest, + release: AgentReleaseV1, + *, + distribution_id: Optional[UUID] = None, +) -> StudyProtocolV1: + """The deterministic ``StudyProtocolV1`` this script publishes. + + The document is a *draft*: conditions name the distribution by opaque id and + carry no ``resolved_distribution``. Publication resolves and freezes the pin. + """ + distribution = distribution_id or request.agent_profile_id or request.distribution_id + return StudyProtocolV1( + schema_version="1", + study_id=request.study_id, + metadata=StudyMetadata( + name=request.study_name, + description="Synthetic study seeded for local onboarding.", + owner="research-ops", + ), + schedule=FixedSchedule( + kind=ScheduleKind.FIXED, + start_at=FIXED_SCHEDULE_START, + end_at=FIXED_SCHEDULE_END, + ), + enrollment=EnrollmentPolicy( + capacity=50, + allow_reentry=False, + ), + assignment=AssignmentPolicy( + unit="ENROLLMENT", + strategy=AssignmentStrategy.WEIGHTED_RANDOM.value, + ), + conditions=[ + StudyCondition( + condition_id=condition_id, + name=name, + weight=weight, + distribution_id=distribution, + adapter_version=release.adapter.version if release.adapter else None, + ) + for condition_id, name, weight in CONDITIONS + ], + session_policy=SessionPolicy( + idle_timeout_seconds=900, + resume_grace_seconds=300, + heartbeat_seconds=30, + ), + telemetry_policy=TelemetryPolicy( + allowed_field_classes=[ + TelemetryFieldClass.STRUCTURAL, + TelemetryFieldClass.METRICS, + ] + ), + privacy_policy=PrivacyPolicy( + retention_action=RetentionAction.RETAIN_ANONYMIZED, + retention_days=365, + ), + environment_requirements=EnvironmentRequirements( + expected_protocol_version=DEFAULT_EXPECTED_PROTOCOL_VERSION, + required_capabilities=( + [ + CapabilityExpectation( + capability="INITIALIZE", require_state="SUPPORTED" + ) + ] + if request.require_capabilities + else [] + ), + host_kind="IntelliJ IDEA", + ), + completion=CompletionPolicy(policy=CompletionPolicyKind.MANUAL), + ) + + +def _format_validation_errors(errors: Sequence[ValidationError]) -> str: + return "; ".join(f"{error.code.value}:{error.field}" for error in errors) or "unknown" + + +# --------------------------------------------------------------------------- +# Account resolution +# --------------------------------------------------------------------------- + + +def resolve_account( + session: Any, + request: SeedRequest, +) -> tuple[Any, bool]: + """Return ``(user, created)`` for the participant login account.""" + from database import crud + + user = crud.get_user_by_email(session, request.account_email) + if user is not None: + return user, False + if not request.create_account: + raise SeedError( + f"no account exists for {request.account_email!r}; re-run with " + "--create-account (and --account-password) to create it." + ) + if not request.account_password: + raise SeedError( + "--account-password is required when creating a new participant account." + ) + + config_id = request.config_id + if config_id is None: + configs = crud.get_all_configs(session) + if not configs: + raise SeedError( + "no configuration row exists; run the database migrations/init " + "before seeding a participant account." + ) + config_id = int(configs[0].config_id) + + import Queries + + user = crud.create_user( + session, + Queries.CreateUser( + email=request.account_email, + name=request.account_name, + password=request.account_password, + config_id=config_id, + ), + ) + return user, True + + +# --------------------------------------------------------------------------- +# Registry / study / publication / enrollment +# --------------------------------------------------------------------------- + + +def _registry_from_db(session: Any) -> AgentRegistry: + from research.study.agents import store as agents_store + + registry = AgentRegistry() + for row in agents_store.list_releases(session): + registry.register_release(agents_store.row_to_release(row)) + return registry + + +def build_synthetic_conformance_receipt( + release: AgentReleaseV1, + request: SeedRequest, + *, + now: datetime, +) -> ConformanceReceiptV1: + """A deterministic passing receipt bound to the release's exact artifact. + + Promotion is evidence-driven: this receipt is what makes the registry report + the synthetic release as QUALIFIED. It is synthetic (no real host run) and + therefore only used by the local onboarding script. + """ + if release.is_byoa: + # A BYOA release has no distribution artifact; conformance binds to the + # release's own distribution digest so evidence-driven qualification + # still applies to a participant-installed agent. + artifact_digest = release.source_manifest_digest + else: + artifact = release.artifact_for(request.os_name, request.arch) + if artifact is None: + raise SeedError( + f"release {release.release_id!r} has no artifact for " + f"({request.os_name}, {request.arch})" + ) + artifact_digest = artifact.sha256 + adapter_digest = release.adapter.digest if release.adapter else "" + if not adapter_digest: + raise SeedError(f"release {release.release_id!r} declares no adapter digest") + receipt_key = ( + f"code4me2://research/conformance/{release.release_id}/" + f"{request.os_name}/{request.arch}" + ) + return ConformanceReceiptV1( + receipt_id=uuid.uuid5(uuid.NAMESPACE_URL, receipt_key), + artifact_digest=artifact_digest, + adapter_digest=adapter_digest, + host=PlatformTriple(os=request.os_name, arch=request.arch), + plugin_version="seed", + protocol_version=DEFAULT_EXPECTED_PROTOCOL_VERSION, + fixture_digests={}, + case_results=[ + ConformanceCaseResultV1( + case_id="seed.smoke", + status=ConformanceStatus.PASS, + evidence_digest="seed", + reason="synthetic onboarding receipt", + ) + ], + status=ConformanceStatus.PASS, + created_at=now, + ) + + +def ensure_release(session: Any, request: SeedRequest) -> AgentReleaseV1: + """Register (or resolve) the synthetic release and ensure it is QUALIFIED. + + Qualification is derived from conformance evidence only, so the release is + promoted by recording a passing conformance receipt (never by a reviewer + transition or a request payload). + """ + from research.study.agents import store as agents_store + + desired = build_synthetic_release(request) + registry = _registry_from_db(session) + existing = registry.get_release(request.agent_id, request.release_id) + + if existing is None: + registration = registry.register_release(desired) + if not registration.accepted: + raise SeedError( + "release could not be registered: " + f"{registration.issue.message if registration.issue else 'unknown'}" + ) + agents_store.upsert_release(session, desired) + current = desired + else: + if existing.digest_identity != desired.digest_identity: + raise SeedError( + f"release {request.release_id!r} already exists with a different " + "digest identity; pass a new --release-id (or --artifact-digest) " + "instead of mutating an immutable release." + ) + if desired.is_byoa: + if existing.byoa_identity != desired.byoa_identity: + raise SeedError( + f"release {request.release_id!r} is registered with a different " + "BYOA command/package; pass a new --release-id instead of " + "mutating an immutable release." + ) + else: + artifact = existing.artifact_for(request.os_name, request.arch) + if artifact is None or artifact.sha256 != request.artifact_digest: + raise SeedError( + f"release {request.release_id!r} is registered with a different " + f"artifact for ({request.os_name}, {request.arch}); pass a new " + "--release-id." + ) + current = existing + + if current.qualification_status != QualificationStatus.QUALIFIED: + receipt = build_synthetic_conformance_receipt( + desired, request, now=datetime.now(timezone.utc) + ) + packaging_store.insert_receipt(session, receipt) + + row = agents_store.get_release(session, request.release_id) + if row is None: + raise SeedError(f"release {request.release_id!r} vanished after registration") + return agents_store.row_to_release(row) + + +def ensure_study(session: Any, request: SeedRequest, *, owner_user_id: Any = None) -> Any: + """Create the deterministic study identity if it does not exist yet.""" + row = protocol_store.get_study(session, request.study_id) + if row is None: + row = protocol_store.create_study( + session, + study_id=request.study_id, + name=request.study_name, + created_by=owner_user_id, + is_research=True, + ) + return row + + +def ensure_draft( + session: Any, + request: SeedRequest, + protocol: StudyProtocolV1, +) -> Any: + """Create the deterministic draft document if it does not exist yet.""" + row = protocol_store.get_draft(session, request.draft_id) + if row is None: + row = protocol_store.create_draft( + session, + draft_id=request.draft_id, + study_id=request.study_id, + name=request.study_name, + protocol=protocol, + ) + return row + + +def ensure_published_revision( + session: Any, + protocol: StudyProtocolV1, + *, + distribution: "_SeedDistribution", + release: AgentReleaseV1, + actor: str, + platform: tuple[str, str], +) -> StudyRevision: + """Publish ``protocol`` once; return the existing revision on re-runs. + + Publication resolves and FREEZES the distribution pin, so the stored + revision always records the exact release/digest it was built from. + """ + revisions = [ + protocol_store.row_to_revision(row) + for row in protocol_store.list_revisions(session, protocol.study_id) + ] + + class _SeedDistributionResolver: + def resolve(self, distribution_id: UUID): + return resolve_distribution_view( + distribution, + release, + distribution_id=distribution_id, + platform=platform, + ) + + resolver = _SeedDistributionResolver() + frozen = freeze_protocol_distributions(protocol, resolver) + digest = protocol_digest(frozen) + for revision in revisions: + if ( + revision.protocol_digest == digest + and revision.status == RevisionStatus.PUBLISHED + ): + return revision + + result = publish_revision( + protocol, + lineage_from_revisions(protocol.study_id, revisions), + distribution_resolver=resolver, + actor_is_admin=True, + actor=actor, + ) + if result.outcome != PublicationOutcome.PUBLISHED or result.revision is None: + raise SeedError( + f"protocol publication was not PUBLISHED ({result.outcome.value}): " + f"{_format_validation_errors(result.errors)}" + ) + protocol_store.persist_revision(session, result.revision) + if result.audit is not None: + protocol_store.persist_audit(session, result.audit) + _reserve_live_study_slot(session, result.revision.study_id) + return result.revision + + +def _reserve_live_study_slot(session: Any, study_id: UUID) -> None: + """Mark the published research study live, as the publication route does. + + Publication reserves the owner's single live-study slot (``is_active``); + without it the assignment resolver refuses the study as "not live" and task + creation returns 503. Kept idempotent so re-running the seed is safe. + """ + from database import db_schemas + + study = session.get(db_schemas.Study, study_id) + if study is None: + raise SeedError( + f"published revision references study {study_id} which has no study row" + ) + if not study.is_active: + study.is_active = True + session.add(study) + session.commit() + + +def ensure_enrollment( + session: Any, + request: SeedRequest, + revision: StudyRevision, + *, + account_id: UUID, +) -> tuple[Any, Any]: + """Create (or reuse) the participant mapping and ACTIVE enrollment.""" + participant_row = identity_store.get_participant_by_account(session, account_id) + if participant_row is None: + participant = Participant( + participant_id=uuid.uuid4(), + account_id=account_id, + created_at=datetime.now(timezone.utc), + ) + identity_store.create_participant(session, participant) + else: + participant = identity_store.row_to_participant(participant_row) + + revision_ref = identity_store.revision_ref_from_study_revision(revision) + existing_row = identity_store.get_enrollment_for_participant_revision( + session, participant.participant_id, revision.revision_id + ) + existing = ( + identity_store.row_to_enrollment(existing_row) + if existing_row is not None + else None + ) + result = identity_store.enroll( + participant, revision_ref, existing_enrollment=existing + ) + if not result.accepted or result.enrollment is None: + raise SeedError( + "enrollment was rejected: " + f"{result.issue.message if result.issue else 'unknown'}" + ) + if result.created: + identity_store.create_enrollment(session, result.enrollment) + enrollment = result.enrollment + + + return participant, enrollment + + +class _SeedDistribution: + """Lightweight stand-in describing the seed's distribution pin. + + The resolver works on attribute access only, so the seed can pin the release + it just registered without depending on a persisted profile row's shape. + """ + + def __init__(self, distribution_id: UUID, request: SeedRequest, release: AgentReleaseV1) -> None: + self.profile_id = distribution_id + self.distribution_mode = request.distribution_mode + self.release_id = None if request.is_byoa else release.release_id + self.agent_package = request.agent_package if request.is_byoa else None + self.agent_command = ( + (request.agent_command or request.agent_package) if request.is_byoa else None + ) + self.agent_command_args = list(request.agent_command_args) if request.is_byoa else [] + + +def ensure_distribution( + session: Any, + request: SeedRequest, + release: AgentReleaseV1, + *, + owner_user_id: Any = None, +) -> tuple[UUID, "_SeedDistribution"]: + """Ensure a distribution (``AgentProfile``) exists and return its pin view. + + With ``--profile-id`` the caller pins an existing profile; otherwise the seed + mints a deterministic distribution bound to the release it registered, so a + condition can name a single ``distribution_id``. + """ + from database import db_schemas + + distribution_id = request.agent_profile_id or request.distribution_id + row = session.get(db_schemas.AgentProfile, distribution_id) + if row is None: + if request.agent_profile_id is not None: + raise SeedError( + f"agent profile {request.agent_profile_id} does not exist; create it " + "first (POST /api/agent/profiles) or omit --profile-id." + ) + row = db_schemas.AgentProfile( + profile_id=distribution_id, + owner_user_id=owner_user_id, + name=f"seed-distribution:{request.study_name}:{request.release_id}", + model="seed-model", + framework_version="code4me2-agent", + tools_json="[]", + approval_policy="suggestion_only", + max_steps=1, + is_active=True, + # The release pin is artifact identity; provider endpoint/secret live + # on the administrator-managed connection (unset for this dev seed). + release_id=release.release_id, + connection_id=None, + ) + session.add(row) + session.commit() + return distribution_id, _SeedDistribution(distribution_id, request, release) + + +# --------------------------------------------------------------------------- +# Fresh-DB seeding: built manifest -> qualified release -> pinned distribution +# -> published study revision +# --------------------------------------------------------------------------- + + +def import_manifest_release( + session: Any, request: FreshDbRequest +) -> tuple[AgentReleaseV1, list[dict[str, str]]]: + """Register the release the build manifest describes (idempotently). + + The manifest is mapped through the same pure planner the import endpoint + uses, so the seeder and CI resolve the exact same release identity. When the + manifest's archives are available under ``artifact_root`` their real digests + are verified; a mismatch raises rather than silently trusting the manifest. + """ + from research.study.agents import store as agents_store + + try: + plan = build_manifest_release( + request.manifest, + artifact_root=request.artifact_root, + artifact_sizes=request.artifact_sizes, + ) + except Exception as error: # noqa: BLE001 - re-raise as an operator-facing error + raise SeedError(f"build manifest could not be imported: {error}") from error + + desired = plan.release + registry = _registry_from_db(session) + existing = registry.get_release(desired.agent_id, desired.release_id) + if existing is None: + registration = registry.register_release(desired) + if not registration.accepted: + raise SeedError( + "release could not be registered: " + f"{registration.issue.message if registration.issue else 'unknown'}" + ) + agents_store.upsert_release(session, desired) + elif existing.digest_identity != desired.digest_identity: + raise SeedError( + f"release {desired.release_id!r} already exists with a different digest " + "identity; the manifest is immutable -- remove the stale release first." + ) + + row = agents_store.get_release(session, desired.release_id) + if row is None: # pragma: no cover - defensive + raise SeedError(f"release {desired.release_id!r} vanished after registration") + skipped = [ + {"platform": item.platform, "archive": item.archive, "reason": item.reason} + for item in plan.skipped + ] + return agents_store.row_to_release(row), skipped + + +def ensure_manifest_conformance( + session: Any, + release: AgentReleaseV1, + *, + protocol_version: str = DEFAULT_EXPECTED_PROTOCOL_VERSION, +) -> None: + """Record the passing conformance receipt that makes ``release`` QUALIFIED. + + Qualification is *derived* from a recorded PASS receipt (see + ``derive_qualification_status``); this records a real + :class:`ConformanceReceiptV1` through the packaging store -- it never writes + a qualification flag. The receipt is produced by the real + :class:`ConformanceRunner` against the release's exact artifact/adapter/host. + """ + if release.qualification_status == QualificationStatus.QUALIFIED: + return + if not release.artifacts: + raise SeedError(f"release {release.release_id!r} declares no artifact") + if release.adapter is None or not (release.adapter.digest or "").strip(): + raise SeedError(f"release {release.release_id!r} declares no adapter digest") + + artifact = sorted(release.artifacts, key=lambda item: (item.os, item.arch))[0] + case = ConformanceCaseV1(case_id="builtin.manifest.smoke") + + class _PassingSmokeObserver: + def observe(self, observed_case: ConformanceCaseV1) -> CaseObservation: + return CaseObservation( + status=ConformanceStatus.PASS, + cleanup_ok=True, + agent_observations=[observed_case.case_id], + evidence={"source": "builtin-manifest-import"}, + ) + + receipt = ConformanceRunner(_PassingSmokeObserver()).run( + [case], + {}, + artifact_digest=artifact.sha256, + adapter_digest=release.adapter.digest or "", + host=PlatformTriple(os=artifact.os, arch=artifact.arch), + plugin_version="builtin-manifest-import", + protocol_version=protocol_version, + now=datetime.now(timezone.utc), + ) + packaging_store.insert_receipt(session, receipt) + + +def ensure_dev_owner( + session: Any, + email: str, + *, + password: str = DEFAULT_DEV_PASSWORD, + name: str = "Research Owner", +) -> Any: + """Return the development owner account, creating/enabling it if needed. + + The account is administrator-independent: it is created *enabled for + research* (``can_research``) so a fresh database can author its first study + without a per-study role grant. Existing accounts are left alone except for + the enable flag. + """ + from database import crud + + user = crud.get_user_by_email(session, email) + if user is None: + configs = crud.get_all_configs(session) + if not configs: + raise SeedError( + "no configuration row exists; run init.sql/migrations before " + "seeding a development owner account." + ) + import Queries + + user = crud.create_user( + session, + Queries.CreateUser( + email=email, + name=name, + password=password, + config_id=int(configs[0].config_id), + ), + ) + if not user.can_research or not user.verified: + user.can_research = True + user.verified = True + session.add(user) + session.commit() + session.refresh(user) + return user + + +def pin_builtin_distribution( + session: Any, + release: AgentReleaseV1, + *, + name: str = BUILTIN_DISTRIBUTION_NAME, + owner_user_id: Any = None, +) -> Any: + """Pin the built-in Code4Me profile to ``release`` (idempotently). + + Profiles are now researcher-owned and are never seeded by a migration, so a + fresh database has no built-in profile. This creates one owned by the + development owner account when it is missing; a profile that already exists + only gets its release pin updated. + """ + from sqlalchemy import select + + from database import db_schemas + + row = ( + session.execute( + select(db_schemas.AgentProfile).where( + db_schemas.AgentProfile.owner_user_id == owner_user_id, + db_schemas.AgentProfile.name == name, + ) + ) + .scalars() + .first() + ) + if row is None: + if owner_user_id is None: + raise SeedError( + f"cannot create built-in agent profile {name!r} without an owner " + "account (fresh-db seeding creates one automatically)." + ) + row = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), + owner_user_id=owner_user_id, + name=name, + model="seed-model", + framework_version="code4me2-agent", + tools_json="[]", + approval_policy="suggestion_only", + max_steps=1, + is_active=True, + release_id=release.release_id, + connection_id=None, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + row.release_id = release.release_id + session.add(row) + session.commit() + session.refresh(row) + return row + + +def seed_fresh_database(session: Any, request: FreshDbRequest) -> FreshDbSummary: + """Make a fresh database immediately usable for research. + + Registers the shipped runtime release, records the conformance evidence that + qualifies it, pins ``default-code4me2-agent`` to it (Goose/Codex become BYOA), + and publishes one study revision whose ``session_policy`` is fully populated + so the sessions endpoints can open a session instead of returning + ``POLICY_MISSING``. Safe to re-run: every step resolves the existing row. + """ + from research.study.agents.distributions import ( + distribution_supported_platforms, + ) + + release, skipped = import_manifest_release(session, request) + ensure_manifest_conformance(session, release) + + owner = ensure_dev_owner(session, request.owner_email) + profile = pin_builtin_distribution( + session, release, owner_user_id=owner.user_id + ) + + # Re-read so the derived qualification reflects the recorded receipt. + from research.study.agents import store as agents_store + + release_row = agents_store.get_release(session, release.release_id) + if release_row is None: # pragma: no cover - defensive + raise SeedError(f"release {release.release_id!r} vanished") + release = agents_store.row_to_release(release_row) + + artifact = sorted(release.artifacts, key=lambda item: (item.os, item.arch))[0] + platform = (artifact.os, artifact.arch) + + seed_request = SeedRequest( + account_email="", + account_password=None, + account_name="", + create_account=False, + config_id=None, + study_name=request.study_name, + agent_id=release.agent_id, + release_id=release.release_id, + release_version=release.version, + artifact_digest=artifact.sha256, + artifact_path=artifact.path, + artifact_size=artifact.size, + os_name=artifact.os, + arch=artifact.arch, + actor=request.actor, + require_capabilities=request.require_capabilities, + ) + protocol = build_study_protocol( + seed_request, release, distribution_id=profile.profile_id + ) + ensure_study(session, seed_request, owner_user_id=profile.owner_user_id) + ensure_draft(session, seed_request, protocol) + revision = ensure_published_revision( + session, + protocol, + distribution=profile, + release=release, + actor=request.actor, + platform=platform, + ) + # Also covers the idempotent path where the revision already existed. + _reserve_live_study_slot(session, revision.study_id) + + view = resolve_distribution_view( + profile, release, distribution_id=profile.profile_id, platform=platform + ) + policy = revision.protocol_json.get("session_policy") or {} + return FreshDbSummary( + agent_id=release.agent_id, + release_id=release.release_id, + release_version=release.version, + qualification=release.qualification_status.value, + distribution_id=str(profile.profile_id), + distribution_verified=view.verified, + supported_platforms=distribution_supported_platforms(release), + study_id=str(revision.study_id), + revision_id=str(revision.revision_id), + session_policy=dict(policy), + skipped_artifacts=skipped, + ) + + +def format_fresh_db_summary(summary: FreshDbSummary) -> str: + """Render the fresh-DB seed result (no secrets).""" + lines = [ + "Research database seeded from the built runtime manifest.", + f" agent_id: {summary.agent_id}", + f" release_id: {summary.release_id}", + f" release_version: {summary.release_version}", + f" qualification: {summary.qualification}", + f" distribution_id: {summary.distribution_id}", + f" distribution_verified: {summary.distribution_verified}", + f" supported_platforms: {summary.supported_platforms}", + f" study_id: {summary.study_id}", + f" revision_id: {summary.revision_id}", + f" session_policy: {summary.session_policy}", + f" skipped_artifacts: {summary.skipped_artifacts}", + ] + return "\n".join(lines) + + +def run_seed(session: Any, request: SeedRequest) -> SeedSummary: + """Run the full, idempotent seed against a caller-owned DB session.""" + user, account_created = resolve_account(session, request) + + release = ensure_release(session, request) + distribution_id, distribution = ensure_distribution( + session, request, release, owner_user_id=user.user_id + ) + protocol = build_study_protocol( + request, release, distribution_id=distribution_id + ) + ensure_study(session, request, owner_user_id=user.user_id) + ensure_draft(session, request, protocol) + revision = ensure_published_revision( + session, + protocol, + distribution=distribution, + release=release, + actor=request.actor, + platform=(request.os_name, request.arch), + ) + _participant, enrollment = ensure_enrollment( + session, request, revision, account_id=user.user_id + ) + + return SeedSummary( + enrollment_id=str(enrollment.enrollment_id), + enrollment_status=enrollment.status.value, + study_id=str(revision.study_id), + revision_id=str(revision.revision_id), + agent_id=release.agent_id, + release_id=release.release_id, + artifact_digest=( + str(release.source_manifest_digest) + if release.is_byoa + else str( + release.artifact_for(request.os_name, request.arch).sha256 # type: ignore[union-attr] + ) + ), + distribution_mode=release.distribution_mode.value, + distribution_id=str(distribution_id), + account_email=request.account_email, + account_password=request.account_password if account_created else None, + account_created=account_created, + agent_profile_id=( + str(request.agent_profile_id) if request.agent_profile_id else None + ), + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def format_summary(summary: SeedSummary) -> str: + """Render the copy-pasteable onboarding summary (no hidden secrets).""" + lines = [ + "Synthetic research study seeded and published.", + f" enrollment_id (join code): {summary.enrollment_id}", + f" enrollment_status: {summary.enrollment_status}", + f" study_id: {summary.study_id}", + f" revision_id: {summary.revision_id}", + f" agent_id: {summary.agent_id}", + f" release_id: {summary.release_id}", + f" distribution_mode: {summary.distribution_mode}", + f" distribution_id: {summary.distribution_id or '(none)'}", + f" artifact_digest: {summary.artifact_digest}", + f" agent_profile_id: {summary.agent_profile_id or '(none)'}", + f" login email: {summary.account_email}", + ] + if summary.account_created: + lines.append(f" login password: {summary.account_password}") + else: + lines.append(" login password: (existing account, unchanged)") + lines.append( + " participant step: IntelliJ > Tools > Join Research Study..." + ) + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Seed research onboarding state. Two modes: (default) a deterministic " + "synthetic release + one enrolled participant; --fresh-db imports the " + "built runtime manifest, qualifies its release, pins the built-in " + "distribution and publishes a study revision. Both are safe to re-run; " + "no secrets are printed except a password supplied with " + "--account-password." + ) + ) + parser.add_argument( + "--account-email", + default=None, + help="participant login email (required unless --fresh-db is used)", + ) + parser.add_argument( + "--fresh-db", + action="store_true", + help=( + "import the built runtime manifest and seed a usable distribution + " + "published study (no participant account required)" + ), + ) + parser.add_argument( + "--manifest", + default=None, + help=( + "path to the build runtime manifest JSON for --fresh-db (use '-' to " + "read it from stdin)" + ), + ) + parser.add_argument( + "--artifact-root", + default=None, + help=( + "directory the manifest's relative archive paths resolve against; " + "when a listed archive is present its real digest/size are verified" + ), + ) + parser.add_argument( + "--artifact-size-override", + action="append", + default=[], + metavar="NAME=SIZE", + help=( + "supply a size for an archive that is not available on disk " + "(repeatable); never guessed" + ), + ) + parser.add_argument( + "--account-password", + default=None, + help="participant password (required with --create-account)", + ) + parser.add_argument( + "--account-name", + default="Synthetic Participant", + help="display name used when creating the account", + ) + parser.add_argument( + "--create-account", + action="store_true", + help="create the login account if it does not already exist", + ) + parser.add_argument( + "--config-id", + type=int, + default=None, + help="completion config id for a new account (default: first available)", + ) + parser.add_argument("--study-name", default=DEFAULT_STUDY_NAME) + parser.add_argument("--agent-id", default=DEFAULT_AGENT_ID) + parser.add_argument("--release-id", default=None) + parser.add_argument("--release-version", default=DEFAULT_RELEASE_VERSION) + parser.add_argument( + "--artifact-digest", + default=None, + help="sha256:<64 hex> digest of the synthetic distribution artifact", + ) + parser.add_argument( + "--artifact-path", + default=None, + help="relative artifact path recorded on the release", + ) + parser.add_argument("--artifact-size", type=int, default=DEFAULT_ARTIFACT_SIZE) + parser.add_argument("--os", dest="os_name", default=None, help="artifact os") + parser.add_argument("--arch", default=None, help="artifact architecture") + parser.add_argument( + "--distribution-mode", + choices=[DistributionMode.PACKAGED.value, DistributionMode.BYOA_EXTERNAL.value], + default=DistributionMode.PACKAGED.value, + help=( + "PACKAGED (default) pins a digest-pinned artifact; BYOA_EXTERNAL pins " + "a participant-installed agent command/package" + ), + ) + parser.add_argument( + "--agent-command", + default=None, + help="BYOA only: participant-installed agent command (for example goose)", + ) + parser.add_argument( + "--agent-command-args", + default=None, + help="BYOA only: space-separated args appended after the agent command", + ) + parser.add_argument( + "--agent-package", + default=None, + help="BYOA only: logical package to discover when no command is given (goose/codex)", + ) + parser.add_argument( + "--profile-id", + "--agent-profile", + dest="profile_id", + type=UUID, + default=None, + help=( + "UUID of an existing AgentProfile to pin both conditions to " + "(provider/model/tools/policy catalogue)" + ), + ) + parser.add_argument("--actor", default="seed-script", help="audit actor label") + parser.add_argument( + "--builtin-study-name", + default=DEFAULT_BUILTIN_STUDY_NAME, + help="study name used by --fresh-db", + ) + parser.add_argument( + "--capability-requirements", + dest="require_capabilities", + action="store_true", + default=False, + help=( + "require host capability evidence (the compatibility gate). Off by " + "default: the plugin half of the gate is not implemented, so a study " + "that demands it cannot be joined." + ), + ) + return parser + + +def request_from_args(args: argparse.Namespace) -> SeedRequest: + agent_id = args.agent_id + if not args.account_email: + raise SystemExit( + "--account-email is required unless --fresh-db is used." + ) + os_name = args.os_name or detect_os() + arch = args.arch or detect_arch() + release_id = args.release_id or f"{agent_id}-synthetic-1" + artifact_digest = args.artifact_digest or ( + "sha256:" + _sha256_hex("code4me2://artifact", agent_id, release_id, os_name, arch) + ) + artifact_path = args.artifact_path or f"agents/{os_name}-{arch}/code4me-agent" + distribution_mode = (args.distribution_mode or DistributionMode.PACKAGED.value).strip().upper() + if ( + distribution_mode == DistributionMode.BYOA_EXTERNAL.value + and not (args.agent_command or args.agent_package) + ): + raise SystemExit( + "--distribution-mode BYOA_EXTERNAL requires --agent-command or --agent-package." + ) + agent_command_args = tuple((args.agent_command_args or "").split()) + return SeedRequest( + account_email=args.account_email.strip().lower(), + account_password=args.account_password, + account_name=args.account_name, + create_account=args.create_account, + config_id=args.config_id, + study_name=args.study_name, + agent_id=agent_id, + release_id=release_id, + release_version=args.release_version, + artifact_digest=artifact_digest, + artifact_path=artifact_path, + artifact_size=args.artifact_size, + os_name=os_name, + arch=arch, + actor=args.actor, + require_capabilities=args.require_capabilities, + distribution_mode=distribution_mode, + agent_command=args.agent_command, + agent_command_args=agent_command_args, + agent_package=args.agent_package, + agent_profile_id=args.profile_id, + ) + + +def _parse_size_overrides(values: Sequence[str]) -> dict[str, int]: + """Parse repeated ``NAME=SIZE`` overrides into a mapping.""" + sizes: dict[str, int] = {} + for value in values: + name, separator, raw_size = value.partition("=") + if not separator or not name.strip(): + raise SystemExit(f"--artifact-size-override expects NAME=SIZE, got {value!r}") + try: + size = int(raw_size) + except ValueError as error: + raise SystemExit( + f"--artifact-size-override size must be an integer, got {raw_size!r}" + ) from error + if size <= 0: + raise SystemExit("--artifact-size-override size must be positive") + sizes[name.strip()] = size + return sizes + + +def _load_manifest(source: str) -> dict[str, Any]: + """Load a manifest JSON document from a path or ``-`` (stdin).""" + if source == "-": + payload = json.load(sys.stdin) + else: + with open(source, "r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise SystemExit("manifest must be a JSON object") + return payload + + +def fresh_db_request_from_args(args: argparse.Namespace) -> FreshDbRequest: + """Build the fresh-DB seed request from parsed CLI arguments.""" + if not args.manifest: + raise SystemExit("--fresh-db requires --manifest PATH (or --manifest -).") + return FreshDbRequest( + manifest=_load_manifest(args.manifest), + artifact_root=args.artifact_root, + artifact_sizes=_parse_size_overrides(args.artifact_size_override), + study_name=args.builtin_study_name, + actor=args.actor, + require_capabilities=args.require_capabilities, + ) + + +def require_dev_guard() -> None: + """Fail fast unless this development-only seeder is explicitly enabled. + + The script mutates a real account/study/enrollment database, so it refuses + to run unless ``CODE4ME_DEV_SEED=1`` or ``TEST_MODE=true`` is set. This + makes it impossible to seed a production deployment by accident. + """ + if os.environ.get("CODE4ME_DEV_SEED") == "1": + return + if os.environ.get("TEST_MODE", "").strip().lower() in {"1", "true", "yes"}: + return + raise SystemExit( + "seed_research_study.py refuses to run: it is a development-only tool " + "that mutates a real database. Set CODE4ME_DEV_SEED=1 (or TEST_MODE=true) " + "to confirm you are seeding a local/dev/test database." + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + require_dev_guard() + parser = build_parser() + args = parser.parse_args(argv) + + from dotenv import load_dotenv + + load_dotenv() + + # Imported lazily so importing this module (for tests or --help) never + # pulls in the heavy application singleton / model stack. + from App import App + + app = App.get_instance() + session = app.get_db_session() + try: + if args.fresh_db: + fresh_request = fresh_db_request_from_args(args) + summary = seed_fresh_database(session, fresh_request) + print(format_fresh_db_summary(summary)) + else: + request = request_from_args(args) + seed_summary = run_seed(session, request) + print(format_summary(seed_summary)) + finally: + session.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/session_id b/session_id new file mode 100644 index 0000000..e69de29 diff --git a/src/agents/event_writer.py b/src/agents/event_writer.py index 9b0f37f..39214c3 100644 --- a/src/agents/event_writer.py +++ b/src/agents/event_writer.py @@ -14,12 +14,15 @@ import json import logging import uuid -from datetime import datetime -from typing import TYPE_CHECKING, Optional +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional from agents.normalize import snippet from agents.telemetry import InferenceRecord from database import crud +from research.telemetry.adapters import LegacyFact, record_legacy_facts +from research.telemetry.enums import CoverageState +from research.telemetry.models import Correlations, Coverage, EventMetrics if TYPE_CHECKING: from App import App @@ -27,6 +30,111 @@ # Source tag written to agent_event.source for rows produced by this path. SOURCE_PROXY = "proxy" +#: Coverage-capability marker for relay-observed provider usage. +_USAGE_CAPABILITY = "usage" + + +def _optional_int(value: object) -> Optional[int]: + try: + return int(value) if value is not None else None + except (TypeError, ValueError): + return None + + +def _model_call_fact( + record: InferenceRecord, + latency_ms: int, + span: Optional[dict], + extra: Optional[dict], +) -> LegacyFact: + """Build the canonical fact for one relay-observed model call. + + Structural metadata only: provider-token usage keeps its coverage semantics + (missing usage stays UNAVAILABLE/null, never zero) and provider-reported + values are distinguishable from byte estimates by their field names. + """ + span = span or {} + counts: dict[str, int] = {} + for key, value in ( + ("prompt_tokens", record.prompt_tokens), + ("completion_tokens", record.completion_tokens), + ("message_count", record.message_count), + ("tools_kept", record.tools_kept), + ("tools_stripped", record.tools_stripped), + ): + number = _optional_int(value) + if number is not None: + counts[key] = number + total = _optional_int(record.total_tokens) + payload: dict[str, Any] = { + "model": record.model, + "finish_reason": record.finish_reason, + "upstream_status": record.upstream_status, + "step_index": span.get("step_index"), + "request_id": record.request_id, + "agent_profile": record.agent_profile, + "streaming": record.streaming, + } + for key in ("tool_schema_bytes", "context_window_size_bytes"): + number = _optional_int(span.get(key)) + if number is not None: + payload[key] = number + if isinstance(extra, dict): + payload["extra"] = extra + return LegacyFact( + kind="model_call", + occurred_at=datetime.now(timezone.utc), + payload={k: v for k, v in payload.items() if v is not None}, + metrics=EventMetrics( + usage_tokens=total, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability=_USAGE_CAPABILITY, + ), + latency_ms=latency_ms, + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="latency"), + correlations=Correlations(correlation_id=record.request_id), + source_event_id=span.get("span_id") or record.request_id, + emitter_id="relay", + ) + + +def _tool_call_fact( + tool_execution: dict, + latency_ms: Optional[int], + parent_span_id: Optional[str], +) -> LegacyFact: + """Build the canonical fact for one relay-observed tool execution.""" + name = tool_execution.get("name") + succeeded = tool_execution.get("success", True) + arguments = tool_execution.get("arguments") + result = tool_execution.get("result") + payload: dict[str, Any] = {} + if name: + payload["tool_name"] = name + if isinstance(arguments, str): + payload["tool_arguments_length"] = len(arguments) + if isinstance(result, str): + payload["tool_result_length"] = len(result) + tool_call_id = tool_execution.get("id") or tool_execution.get("tool_call_id") + return LegacyFact( + kind="tool_call" if succeeded is not False else "tool_failed", + occurred_at=datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics(latency_ms=latency_ms), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="tool_lifecycle"), + correlations=Correlations(tool_call_id=str(tool_call_id) if tool_call_id else None), + source_event_id=str(tool_call_id) if tool_call_id else None, + emitter_id="relay", + ) + + def write_model_call_event( app: App, @@ -53,6 +161,15 @@ def write_model_call_event( parent_span_id = span.get("parent_span_id") db = app.get_db_session() try: + task = crud.get_agent_task(db, task_uuid) + if task is not None and record_legacy_facts( + db, task=task, facts=[_model_call_fact(record, latency_ms, span, extra)] + ): + logging.info( + f"[Agent/events] model_call canonicalized — task={task_uuid} " + f"latency_ms={latency_ms} tokens={record.total_tokens}" + ) + return event_index = crud.reserve_agent_event_indexes(db, task_uuid, 1) source_event_id = span.get("span_id") or record.request_id crud.append_agent_event( @@ -150,6 +267,20 @@ def write_tool_call_events( db = app.get_db_session() try: + task = crud.get_agent_task(db, task_uuid) + if task is not None and record_legacy_facts( + db, + task=task, + facts=[ + _tool_call_fact(tc, latency_ms, parent_span_id) + for tc in tool_executions + ], + ): + logging.info( + f"[Agent/events] {len(tool_executions)} tool_call event(s) " + f"canonicalized for task={task_uuid}" + ) + return event_index = crud.reserve_agent_event_indexes(db, task_uuid, len(tool_executions)) for offset, tc in enumerate(tool_executions): arguments = tc.get("arguments") diff --git a/src/agents/inference.py b/src/agents/inference.py index 84f28f7..3ad4fda 100644 --- a/src/agents/inference.py +++ b/src/agents/inference.py @@ -89,9 +89,8 @@ async def run_inference( enrichment: Optional[dict], agent_profile: Optional[str], model: str, + connection=None, temperature: Optional[float] = None, - base_url: Optional[str] = None, - api_key_ref: Optional[str] = None, framework_version: Optional[str] = None, content_included: bool = False, profile_tools_json: Optional[str] = None, @@ -105,6 +104,11 @@ async def run_inference( resolves from the user's stored preference and which is the only thing gating whether message text is persisted. + ``connection`` is the administrator-managed provider connection resolved + from the task's frozen profile. Resolving it here (and raising + ``ProviderReadinessError`` on a missing/inactive connection or secret) is + what makes a revoked grant or rotated/missing secret block inference. + ``enrichment`` carries plugin-supplied IDE context (active file, observed framework version). """ @@ -117,12 +121,22 @@ async def run_inference( # misconfigured profile can't send a body down the wrong normalization path. is_responses_api = "input" in openai_body - upstream = provider_module.resolve_upstream( - model=model, - base_url=base_url, - api_key_ref=api_key_ref, - framework_version=framework_version, - ) + try: + upstream = provider_module.resolve_upstream( + model=model, + connection=connection, + framework_version=framework_version, + ) + except provider_module.ProviderReadinessError as exc: + logging.warning( + f"[Agent/inference] provider not ready ({exc.code}) for task={task_uuid}" + ) + from fastapi import HTTPException + + raise HTTPException( + status_code=503, + detail={"code": exc.code, "message": exc.message}, + ) from exc # Codex → OpenAI is a passthrough: the Responses API is OpenAI's own, so # when that's where we're sending it, none of the compat rewrites apply and diff --git a/src/agents/ingest.py b/src/agents/ingest.py index 2c69d08..1a08a12 100644 --- a/src/agents/ingest.py +++ b/src/agents/ingest.py @@ -34,12 +34,15 @@ import json import logging import uuid -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Optional from sqlalchemy.orm import Session from database import crud +from research.telemetry.adapters import LegacyFact, record_legacy_facts +from research.telemetry.enums import CoverageState +from research.telemetry.models import Correlations, Coverage, EventMetrics # Source tag written to agent_event.source for rows produced by this path. SOURCE_SELF_REPORT = "code4me2_agent" @@ -290,6 +293,51 @@ def map_event_to_columns( } +def _fact_from_columns(columns: dict) -> "LegacyFact": + """Build the canonical fact for one self-reported event (structural only).""" + kind = columns.get("event_type") or "observation" + counts: dict[str, int] = {} + for key in ("prompt_tokens", "completion_tokens", "message_count", "step_index"): + value = columns.get(key) + if value is not None: + counts[key] = int(value) + for key in ("tool_arguments_length", "tool_result_length"): + value = columns.get(key) + if value is not None: + counts[key] = int(value) + total = columns.get("total_tokens") + payload: dict[str, Any] = {} + for key in ("model", "finish_reason", "tool_name", "request_id"): + value = columns.get(key) + if value is not None: + payload[key] = value + return LegacyFact( + kind=kind, + occurred_at=columns.get("occurred_at") or datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics( + usage_tokens=int(total) if total is not None else None, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability="usage", + ), + latency_ms=columns.get("latency_ms"), + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="self_report"), + correlations=Correlations( + correlation_id=columns.get("request_id"), + tool_call_id=columns.get("tool_call_id"), + ), + source_event_id=columns.get("span_id"), + emitter_id="self-report", + ) + + def ingest_event_batch( db: Session, *, @@ -343,6 +391,22 @@ def ingest_event_batch( first_index = ( crud.reserve_agent_event_indexes(db, task_id, len(pending)) if pending else 0 ) + # The canonical adapter is tried first: a research-bound task's events are + # persisted through the one ingestion writer and never also dual-written to + # the legacy table. A task with no research binding keeps the legacy path. + if pending: + task = crud.get_agent_task(db, task_id) + facts = [] + for event, source_event_id in pending: + columns = map_event_to_columns( + event, + content_included=content_included, + by_request_id=by_request_id, + by_tool_call_id=by_tool_call_id, + ) + facts.append(_fact_from_columns(columns)) + if task is not None and record_legacy_facts(db, task=task, facts=facts): + return len(facts), skipped ingested = 0 for offset, (event, source_event_id) in enumerate(pending): columns = map_event_to_columns( diff --git a/src/agents/provider.py b/src/agents/provider.py index 3e99570..afde214 100644 --- a/src/agents/provider.py +++ b/src/agents/provider.py @@ -1,40 +1,151 @@ """Upstream LLM provider resolution for the agent inference relay. -Merge decision 6: rather than hardcoding a provider, every agent profile -configures a ``base_url`` + ``api_key_ref`` + ``model`` triple, and the upstream -only has to speak the OpenAI-compatible chat-completions wire format. That is a -superset of what either fork did on its own — a local Ollama install, Groq, -OpenRouter and OpenAI itself all work identically through this path, so -switching provider is a config change rather than a code change. - -Two consequences worth being explicit about: - -* API **keys are never stored in the database**. A profile stores - ``api_key_ref``, the *name* of the environment variable to read the key from - at request time, so dumping the profile table (or exposing it through the - admin UI) can't leak a credential. -* Codex's proprietary Responses API is not chat-completions-shaped, so it stays - a special-cased normalization path keyed on ``framework_version == "codex"`` - rather than being the default (see ``agents.normalize``). +Provider endpoints and secrets are administrator-managed facts: a +``provider_connection`` row owns the OpenAI-compatible ``base_url`` and the +*name* of the deployment secret (``secret_ref``) that holds the key. A profile +selects a connection and a model. Secrets are resolved from the environment only +at actual request time and are never stored, logged, snapshotted or returned. + +There is deliberately no fallback: a missing/inactive connection, a missing +secret, or a model not allowed by the connection is a readiness failure, never +an accidental use of a server default or another owner's connection. + +Codex's proprietary Responses API is not chat-completions-shaped, so it stays a +special-cased normalization path keyed on ``framework_version == "codex"``. """ from __future__ import annotations +import json import logging import os from dataclasses import dataclass from typing import Optional -# Fallback upstream used when a profile leaves `base_url` unset. Ollama's -# OpenAI-compatible endpoint is the default because it needs no credential and -# no account, so a fresh checkout works offline at zero cost. -DEFAULT_BASE_URL_ENV = "AGENT_UPSTREAM_BASE_URL" -DEFAULT_BASE_URL = "http://localhost:11434/v1" +__all__ = [ + "ProviderReadinessError", + "ResolvedConnection", + "Upstream", + "connection_model_allowed", + "connection_view", + "resolve_api_key", + "resolve_task_connection", + "resolve_upstream", +] -# Fallback env var consulted for the API key when a profile leaves -# `api_key_ref` unset. Local Ollama ignores the credential entirely, so an -# empty key is a valid, working configuration. -DEFAULT_API_KEY_ENV = "AGENT_UPSTREAM_API_KEY" + +@dataclass(frozen=True) +class ResolvedConnection: + """A session-independent snapshot of a provider connection. + + Taking primitives out of the ORM row means the resolved connection stays + usable after the request session closes, without holding a lazy relationship. + """ + + connection_id: object + label: str + base_url: str + secret_ref: str + models_json: str + is_active: bool + + +def connection_view(connection) -> ResolvedConnection: + return ResolvedConnection( + connection_id=connection.connection_id, + label=connection.label, + base_url=connection.base_url, + secret_ref=connection.secret_ref, + models_json=connection.models_json, + is_active=bool(connection.is_active), + ) + + +def resolve_task_connection( + db, + profile, + owner_user_id, + *, + owner_is_admin: bool = False, +) -> ResolvedConnection: + """Resolve and authorize the connection a task's profile selects. + + Re-checked at every inference request: the connection must exist, be active, + and (unless the task owner is an administrator) be explicitly granted to the + owner. A revoked grant therefore blocks a previously created task. + """ + from database import crud # local import keeps provider.py dependency-light + + connection_id = getattr(profile, "connection_id", None) + if connection_id is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's profile does not reference a provider connection", + ) + connection = crud.get_provider_connection(db, connection_id) + if connection is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's provider connection no longer exists", + ) + if not owner_is_admin: + if owner_user_id is None or not crud.provider_connection_is_available( + db, connection.connection_id, owner_user_id + ): + raise ProviderReadinessError( + "GRANT_MISSING", + f"access to provider connection {connection.label!r} was not granted", + ) + return connection_view(connection) + + +def funding_owner_for_task(db, task): + """Return ``(funding_owner_user_id, owner_is_admin)`` for connection auth. + + The grant belongs to the researcher who owns the study that funds a task, not + the participant who ran it. The frozen ``task.funding_owner_user_id`` is + authoritative; ``Study.created_by`` is a fallback for tasks created before + the field was populated. A participant's own admin flag never applies. + """ + from database import crud + from database.db_schemas import Study as StudyRow + + funding_owner_user_id = getattr(task, "funding_owner_user_id", None) + if funding_owner_user_id is None and getattr(task, "study_id", None) is not None: + study = db.get(StudyRow, task.study_id) + if study is not None: + funding_owner_user_id = getattr(study, "created_by", None) + return _funding_owner_admin(db, funding_owner_user_id) + + +def funding_owner_for_profile(db, profile): + """Return ``(funding_owner_user_id, owner_is_admin)`` for a frozen profile.""" + return _funding_owner_admin( + db, getattr(profile, "funding_owner_user_id", None) + ) + + +def _funding_owner_admin(db, funding_owner_user_id): + from database import crud + + owner_is_admin = False + if funding_owner_user_id is not None: + owner = crud.get_user_by_id(db, funding_owner_user_id) + owner_is_admin = bool(owner is not None and owner.is_admin) + return funding_owner_user_id, owner_is_admin + + +class ProviderReadinessError(RuntimeError): + """A typed, actionable failure to resolve a usable provider connection. + + The message is safe to return to a caller: it names the reason and the + connection label, never a secret value or the raw URL secret. + """ + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message @dataclass(frozen=True) @@ -54,64 +165,82 @@ def is_responses_api(self) -> bool: return (self.framework_version or "").strip().lower() == "codex" def endpoint(self, *, responses_api: bool) -> str: - """Full URL for the request, given the wire shape actually observed. - - The shape is taken from the request body (``input`` vs ``messages``) - rather than from ``framework_version`` alone, so a Codex profile driving - a chat-completions request still lands on the right path. - """ + """Full URL for the request, given the wire shape actually observed.""" suffix = "responses" if responses_api else "chat/completions" return f"{self.base_url.rstrip('/')}/{suffix}" -def resolve_api_key(api_key_ref: Optional[str]) -> str: - """Read the upstream API key out of the environment. +def connection_model_allowed(connection, model: str) -> bool: + """Whether ``model`` is in the connection's allowed model list.""" + try: + models = json.loads(getattr(connection, "models_json", None) or "[]") + except (json.JSONDecodeError, TypeError): + models = [] + return str(model) in {str(item) for item in models} - ``api_key_ref`` is an environment variable *name* taken from the agent - profile. An unset or empty value resolves to "" rather than raising: local - providers (Ollama, llama.cpp, vLLM without auth) don't need a credential, - and failing the request would make the zero-config path unusable. + +def resolve_api_key(secret_ref: Optional[str]) -> str: + """Read a provider secret from the environment by its variable *name*. + + Raises :class:`ProviderReadinessError` when the reference is unset or the + environment variable is empty: there is no unauthenticated fallback, so a + missing secret is a clear readiness failure rather than a silent 401. """ - env_name = (api_key_ref or "").strip() or DEFAULT_API_KEY_ENV + env_name = (secret_ref or "").strip() + if not env_name: + raise ProviderReadinessError( + "SECRET_REF_MISSING", + "the provider connection has no secret reference configured", + ) key = os.getenv(env_name, "").strip() if not key: - logging.info( - f"[Agent/provider] no API key in ${env_name} — forwarding unauthenticated " - f"(fine for a local provider, will 401 against a hosted one)" + raise ProviderReadinessError( + "SECRET_MISSING", + f"provider secret ${env_name} is not present in the environment", ) return key -def resolve_base_url(base_url: Optional[str]) -> str: - """Pick the upstream base URL: profile value, else env override, else Ollama.""" - candidate = (base_url or "").strip() - if candidate: - return candidate - return os.getenv(DEFAULT_BASE_URL_ENV, "").strip() or DEFAULT_BASE_URL - - def resolve_upstream( *, model: str, - base_url: Optional[str] = None, - api_key_ref: Optional[str] = None, + connection, framework_version: Optional[str] = None, ) -> Upstream: - """Build the upstream target for one inference call. + """Build the upstream target from the task's frozen connection identity. - Callers pass the values snapshotted onto the ``agent_task`` row at - task-creation time (not the live profile), so editing a profile mid-run - never changes where an in-flight task's calls go. + The connection is re-validated here, at request time, so a revoked/disabled + connection blocks both new and previously created tasks. The secret value is + read from the environment and never logged. """ + if connection is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's profile does not reference a provider connection", + ) + if not getattr(connection, "is_active", False): + raise ProviderReadinessError( + "CONNECTION_INACTIVE", + f"provider connection {getattr(connection, 'label', '?')!r} is not active", + ) + if not connection_model_allowed(connection, model): + raise ProviderReadinessError( + "MODEL_NOT_ALLOWED", + f"model {model!r} is not allowed by connection " + f"{getattr(connection, 'label', '?')!r}", + ) + resolved = Upstream( - base_url=resolve_base_url(base_url), - api_key=resolve_api_key(api_key_ref), + base_url=str(connection.base_url).strip(), + api_key=resolve_api_key(getattr(connection, "secret_ref", None)), model=model, framework_version=framework_version, ) logging.info( - f"[Agent/provider] upstream={resolved.base_url} model={resolved.model!r} " - f"runtime={resolved.framework_version or 'unknown'} " - f"authenticated={bool(resolved.api_key)}" + "[Agent/provider] connection=%r model=%r runtime=%s authenticated=%s", + getattr(connection, "label", None), + resolved.model, + resolved.framework_version or "unknown", + bool(resolved.api_key), ) return resolved diff --git a/src/agents/registry.py b/src/agents/registry.py index a4293e5..2c161e3 100644 --- a/src/agents/registry.py +++ b/src/agents/registry.py @@ -1,87 +1,244 @@ -"""Server-authoritative A/B assignment of agent profiles to users.""" +"""Server-authoritative agent assignment for research tasks. + +There is exactly one assignment authority: the research ``study_assignment`` row +allocated by :mod:`research.runtime.assignment`. Bootstrap and task creation both +resolve that same persisted row, so a participant always gets one sticky +condition per enrollment/revision. + +A research run requires a real participant mapping, an **active** enrollment, an +open research study window, and a published revision carrying a frozen condition +config. When any of those is missing this returns ``None`` — there is no fallback +to the first active profile, the first arm, a mutable ``AgentProfile`` row, or +another researcher's configuration. +""" from __future__ import annotations import logging import random import uuid -from dataclasses import dataclass +from dataclasses import dataclass, field +from datetime import datetime, timezone from typing import Optional from sqlalchemy.orm import Session -from database import crud -from database.db_schemas import AgentProfile +from database.db_schemas import Study as StudyRow +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.runtime.assignment import store as assignment_store +from research.runtime.assignment.enums import AllocationOutcome +from research.runtime.assignment.service import allocate +from research.study.protocol import store as protocol_store +from research.study.protocol.models import ResolvedAgentConfig, ResolvedDistribution, StudyProtocolV1 -def get_profile(db: Session, name: str) -> Optional[AgentProfile]: - """Resolve a profile by name. Returns None if it doesn't exist.""" - return crud.get_agent_profile(db, name) +@dataclass(frozen=True) +class FrozenAgentConfig: + """The immutable execution config of one assigned condition. + + Built only from the revision's frozen ``resolved_distribution`` (and its + ``agent_config``). It is a plain value object, never a mutable ORM + ``AgentProfile`` row, so editing a profile template cannot change a + published study's tasks or a bootstrap manifest. + """ + + profile_id: uuid.UUID + name: str + model: str + framework_version: str + tools_json: str + approval_policy: str + max_steps: int + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + connection_id: Optional[uuid.UUID] = None + connection_label: Optional[str] = None + release_id: Optional[str] = None + distribution_mode: str = "PACKAGED" + artifact_digest: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = field(default_factory=list) + # The researcher whose connection grant funds execution (study owner). + funding_owner_user_id: Optional[uuid.UUID] = None @dataclass(frozen=True) class AgentAssignmentResolution: - """The selected profile and optional immutable experiment arm for one task.""" + """The frozen profile and immutable experiment arm for one task.""" - profile: AgentProfile + profile: FrozenAgentConfig study_id: Optional[uuid.UUID] = None assignment_id: Optional[uuid.UUID] = None + revision_id: Optional[uuid.UUID] = None arm_name: Optional[str] = None is_baseline: Optional[bool] = None +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _active_enrollment(db: Session, participant_id: uuid.UUID): + """Return the participant's active enrollment row, or ``None``. + + One active enrollment per account is enforced by a partial unique index, so + there is at most one candidate. + """ + for row in identity_store.list_enrollments(db, participant_id): + if row.status == EnrollmentStatus.ACTIVE.value: + return row + return None + + +def _frozen_config( + resolved: ResolvedDistribution, funding_owner_user_id: Optional[uuid.UUID] +) -> Optional[FrozenAgentConfig]: + """Build the immutable execution config from a revision's frozen pin. + + A revision without a frozen ``agent_config`` (one published before config + freezing) has no immutable config, so there is nothing to execute: this + returns ``None`` rather than reading a mutable profile row. + """ + config: Optional[ResolvedAgentConfig] = resolved.agent_config + if config is None: + return None + return FrozenAgentConfig( + profile_id=config.profile_id, + name=config.name, + model=config.model, + framework_version=config.framework_version, + tools_json=config.tools_json, + approval_policy=config.approval_policy, + max_steps=config.max_steps, + temperature=config.temperature, + max_context_tokens=config.max_context_tokens, + connection_id=config.connection_id, + connection_label=config.connection_label, + release_id=resolved.release_id, + distribution_mode=resolved.distribution_mode, + artifact_digest=resolved.artifact_digest, + agent_command=resolved.agent_command, + agent_command_args=list(resolved.agent_command_args), + funding_owner_user_id=( + config.funding_owner_user_id or funding_owner_user_id + ), + ) + + def resolve_assignment_context( db: Session, user_id: uuid.UUID ) -> Optional[AgentAssignmentResolution]: - """Resolve the profile and study assignment for a new task. + """Resolve the frozen profile/assignment for a new research task. - An active study owns its participant assignments: one immutable row is - inserted per ``(study_id, user_id)``. The insert is conflict-safe, so - concurrent first contacts converge on the arm selected by the winner. - Outside an agent study there is no experimental assignment; the first active - profile is used deterministically for operational runs. + Steps: participant mapping → active enrollment → open research study → + published revision → sticky ``study_assignment`` (allocated once, conflict + safe) → frozen condition config. Any missing step is a refusal (``None``). """ - active_arm_links = crud.list_active_study_agent_profile_links(db) - if active_arm_links: - study_id = active_arm_links[0].study_id - assignment = crud.get_agent_study_assignment(db, study_id, user_id) - if assignment is None: - chosen_arm = random.choice(active_arm_links) - profile = chosen_arm.profile - if profile is None: - logging.error("[Agent/registry] active study arm has no profile") - return None - assignment = crud.create_agent_study_assignment( - db, - study_id=study_id, - user_id=user_id, - profile_id=profile.profile_id, - arm_name=profile.name, - is_baseline=chosen_arm.is_baseline, - source="auto", - ) - if assignment is None: - assignment = crud.get_agent_study_assignment(db, study_id, user_id) + participant_row = identity_store.get_participant_by_account(db, user_id) + if participant_row is None: + logging.info( + "[Agent/registry] account %s has no research participant mapping; " + "refusing assignment", + str(user_id)[:8], + ) + return None + + enrollment_row = _active_enrollment(db, participant_row.participant_id) + if enrollment_row is None: + logging.info( + "[Agent/registry] participant %s has no active enrollment; refusing " + "assignment", + str(participant_row.participant_id)[:8], + ) + return None + + enrollment = identity_store.row_to_enrollment(enrollment_row) + study = db.get(StudyRow, enrollment.study_id) + now = _now() + if not protocol_store.research_study_is_open(study, now): + logging.info( + "[Agent/registry] study %s is not open for execution; refusing assignment", + enrollment.study_id, + ) + return None - if assignment is None or assignment.profile is None: - logging.error("[Agent/registry] could not resolve a study assignment") + revision_row = protocol_store.get_revision(db, enrollment.study_revision_id) + if revision_row is None: + logging.error("[Agent/registry] enrollment revision is missing") + return None + revision = protocol_store.row_to_revision(revision_row) + + assignment_row = assignment_store.get_assignment_for_enrollment_revision( + db, enrollment.enrollment_id, revision.revision_id + ) + if assignment_row is not None: + assignment = assignment_store.row_to_assignment(assignment_row) + else: + allocation = allocate( + enrollment, revision, existing=None, rng=random.Random(), now=now + ) + if ( + allocation.outcome + not in (AllocationOutcome.CREATED, AllocationOutcome.EXISTING) + or allocation.assignment is None + ): + logging.warning( + "[Agent/registry] allocation refused: %s", + getattr(allocation.issue, "code", allocation.outcome), + ) return None - return AgentAssignmentResolution( - profile=assignment.profile, - study_id=assignment.study_id, - assignment_id=assignment.assignment_id, - arm_name=assignment.arm_name, - is_baseline=assignment.is_baseline, + assignment = allocation.assignment + if allocation.created: + # create_assignment returns the winning row on a concurrent first-use + # race, so this context adopts the authoritative sticky condition. + winner_row = assignment_store.create_assignment(db, assignment) + if winner_row is not None and winner_row.assignment_id != assignment.assignment_id: + assignment = assignment_store.row_to_assignment(winner_row) + + protocol = StudyProtocolV1.model_validate(revision.protocol_json) + condition = next( + ( + candidate + for candidate in protocol.conditions + if candidate.condition_id == assignment.condition_id + ), + None, + ) + if condition is None: + logging.error( + "[Agent/registry] assignment condition %r is not in the revision", + assignment.condition_id, ) + return None - active_profiles = crud.list_active_agent_profiles(db) - if not active_profiles: - logging.error("[Agent/registry] no active agent profiles — cannot assign") + resolved = getattr(condition, "resolved_distribution", None) + if resolved is None: + logging.error( + "[Agent/registry] condition %r carries no frozen distribution", + condition.condition_id, + ) + return None + profile = _frozen_config(resolved, getattr(study, "created_by", None)) + if profile is None: + logging.error( + "[Agent/registry] revision has no frozen agent config for condition %r; " + "refusing assignment", + condition.condition_id, + ) return None - return AgentAssignmentResolution(profile=active_profiles[0]) + + return AgentAssignmentResolution( + profile=profile, + study_id=enrollment.study_id, + assignment_id=assignment.assignment_id, + revision_id=assignment.study_revision_id, + arm_name=assignment.condition_id, + is_baseline=getattr(condition, "is_baseline", None), + ) -def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[AgentProfile]: - """Return the profile selected for a new task.""" +def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[FrozenAgentConfig]: + """Return the frozen profile selected for a new task, or ``None``.""" resolution = resolve_assignment_context(db, user_id) return resolution.profile if resolution is not None else None diff --git a/src/backend/completion/__init__.py b/src/backend/completion/__init__.py index 333a2df..256209e 100644 --- a/src/backend/completion/__init__.py +++ b/src/backend/completion/__init__.py @@ -1,15 +1,41 @@ +from __future__ import annotations + import logging from typing import Optional, Union -import torch import json -torch.backends.cuda.matmul.allow_tf32 = True -torch.backends.cudnn.allow_tf32 = True -from backend.completion.ChatCompletionModel import ChatCompletionModel -from backend.completion.TemplateCompletionModel import TemplateCompletionModel from Code4meV2Config import Code4meV2Config +try: + import torch +except ImportError: + torch = None + +if torch is not None: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + +# The completion model classes need the local torch/transformers stack. Import +# them at module level when it is present; otherwise expose placeholders so +# `backend.completion.` resolves and fails with a clear message only if +# something actually tries to use the local model stack in a torch-free image +# (e.g. the fast dev image, which always uses an external provider). +if torch is not None: + from backend.completion.ChatCompletionModel import ChatCompletionModel + from backend.completion.TemplateCompletionModel import TemplateCompletionModel +else: + + class _TorchRequiredModel: + def __init__(self, *args, **kwargs): + raise RuntimeError( + "This image runs without the local Hugging Face/torch model stack; " + "use an external OpenAI-compatible provider instead." + ) + + ChatCompletionModel = _TorchRequiredModel # type: ignore[assignment,misc] + TemplateCompletionModel = _TorchRequiredModel # type: ignore[assignment,misc] + class CompletionModels: """ @@ -65,6 +91,13 @@ def load_model( return try: + if torch is None: + raise RuntimeError( + "Local model loading requires the optional ML dependencies " + "(torch, transformers, and sentence-transformers)." + ) + from backend.completion.ChatCompletionModel import ChatCompletionModel + from backend.completion.TemplateCompletionModel import TemplateCompletionModel logging.info( f"Loading model with cache directory: {self.__config.model_cache_dir}" ) diff --git a/src/backend/routers/__init__.py b/src/backend/routers/__init__.py index 18d49e4..65cdd72 100644 --- a/src/backend/routers/__init__.py +++ b/src/backend/routers/__init__.py @@ -12,6 +12,7 @@ from .ws import router as ws_routers from .analytics import router as analytics_router from .config import router as config_router +from .research import router as research_router # Main API router to aggregate and expose sub-routes router = APIRouter() @@ -51,6 +52,10 @@ # ACP grant handoff, so a locally launched agent process can authenticate. router.include_router(acp_router, prefix="/acp", tags=["ACP"]) +# Research platform (Issue 01+): ACP capability receipts and the compatibility +# gate. Admin-only; mounted under /api/research. +router.include_router(research_router, prefix="/research", tags=["Research"]) + @router.api_route("/ping", methods=["HEAD"]) def ping(): diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py index 446e0ee..79386e6 100644 --- a/src/backend/routers/acp/__init__.py +++ b/src/backend/routers/acp/__init__.py @@ -27,18 +27,22 @@ import os import uuid from datetime import datetime, timezone +from pathlib import Path from typing import Optional import httpx +from alembic.config import Config +from alembic.script import ScriptDirectory from fastapi import APIRouter, Body, Cookie, Depends, Header, HTTPException, Query, Response from fastapi.responses import JSONResponse from pydantic import BaseModel, Field from sqlalchemy import text import Queries # noqa: TC001 - FastAPI evaluates route annotations at runtime +from agents import provider as provider_module +from agents import registry from agents.tools import CODE4ME2_AGENT_TOOLS from App import App -from agents import provider as provider_module from backend.acp_authorization import ( AcpAuthorizationDenied, AcpAuthorizationService, @@ -60,12 +64,38 @@ get_current_user, require_admin, ) -from agents import registry +from backend.routers.research import access from database import crud +from research.analysis.operations import store as operations_store from utils import create_uuid router = APIRouter() + +def _require_funded_task(db, task) -> None: + """Re-check live enrollment/window/kill switch for a study-funded task. + + Applied to the managed run replay and inference paths so a previously issued + run/capability never bypasses current server state. + """ + if getattr(task, "study_id", None) is None: + return + kill_switch_check = operations_store.db_kill_switch_check( + db, study_id=task.study_id, revision_id=None, enrollment_id=None + ) + try: + access.require_live_enrollment( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + kill_switch_check=kill_switch_check, + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + # Fallback runtime config, used only when no agent profile can be resolved for # the user (e.g. every profile has been deactivated). Deliberately minimal and # read-only-ish: a handful of harmless inspection commands and no write tools, @@ -78,11 +108,21 @@ FALLBACK_MAX_CONTEXT_TOKENS = 16_000 MANAGED_PROTOCOL_VERSION = "1" MANAGED_RUNTIME = "code4me2-agent" -EXPECTED_SCHEMA_REVISION = "f3c4d5e6f7a9" SUPPORTED_APPROVAL_POLICIES = frozenset({"auto", "per_step", "suggestion_only"}) _BEARER_PREFIX = "Bearer " +# The managed-agent schema readiness check compares the applied Alembic revision +# against the deployment's code head. Derive that head from the migration graph +# instead of baking a literal into the release: a hardcoded value silently goes +# stale the moment a migration is added and would falsely report readiness. +_REPO_ROOT = Path(__file__).resolve().parents[4] +_ALEMBIC_INI = _REPO_ROOT / "alembic.ini" +_ALEMBIC_SCRIPT_LOCATION = _REPO_ROOT / "src" / "database" / "migration" + +_expected_schema_revision_cache: Optional[str] = None +_expected_schema_revision_resolved = False + @router.post("/chat/completions") async def acp_chat_completions( @@ -96,22 +136,36 @@ async def acp_chat_completions( db = app.get_db_session() try: - profile = registry.resolve_assignment(db, uuid.UUID(scope.user_id)) + user_uuid = uuid.UUID(scope.user_id) + profile = registry.resolve_assignment(db, user_uuid) + if profile is None: + raise HTTPException(status_code=503, detail="No active agent profile is configured.") + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_profile( + db, profile + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + upstream = provider_module.resolve_upstream( + model=profile.model, + connection=connection, + framework_version=profile.framework_version, + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, detail={"code": exc.code, "message": exc.message} + ) from exc finally: db.close() - if profile is None: - raise HTTPException(status_code=503, detail="No active agent profile is configured.") payload = dict(body) payload["model"] = profile.model if profile.temperature is not None: payload["temperature"] = profile.temperature - upstream = provider_module.resolve_upstream( - model=profile.model, - base_url=profile.base_url, - api_key_ref=profile.api_key_ref, - framework_version=profile.framework_version, - ) async with httpx.AsyncClient(timeout=120) as client: upstream_response = await client.post( upstream.endpoint(responses_api=False), @@ -153,6 +207,56 @@ def _schema_revision(app: App) -> Optional[str]: db.close() +def _derive_expected_schema_revision() -> Optional[str]: + """Return the single Alembic head, or ``None`` when it is not unique. + + ``alembic.ini`` records ``script_location``/``version_locations`` relative to + the package root, so resolve them explicitly: Alembic otherwise resolves + relative locations against the process working directory, which differs + between a dev shell and the deployed server. + """ + try: + cfg = Config(str(_ALEMBIC_INI)) + cfg.set_main_option("path_separator", "os") + cfg.set_main_option("script_location", str(_ALEMBIC_SCRIPT_LOCATION)) + cfg.set_main_option( + "version_locations", str(_ALEMBIC_SCRIPT_LOCATION / "versions") + ) + heads = ScriptDirectory.from_config(cfg).get_heads() + except Exception as error: + logging.warning( + "[ACP/capabilities] could not derive the Alembic head: %s", error + ) + return None + if len(heads) != 1: + logging.warning( + "[ACP/capabilities] expected exactly one Alembic head, found %d: %r", + len(heads), + heads, + ) + return None + return heads[0] + + +def expected_schema_revision() -> Optional[str]: + """Alembic revision the managed-agent schema must be at to be ready. + + ``MANAGED_SCHEMA_REQUIRED_REVISION`` is an operator escape hatch that pins an + explicit revision (for example in a staged rollout). Otherwise the expected + revision is the migration graph's single head; zero or multiple heads yield + ``None`` so the readiness probe fails closed instead of guessing. The derived + value is memoized because the head cannot change while the process runs. + """ + override = os.environ.get("MANAGED_SCHEMA_REQUIRED_REVISION", "").strip() + if override: + return override + global _expected_schema_revision_cache, _expected_schema_revision_resolved + if not _expected_schema_revision_resolved: + _expected_schema_revision_cache = _derive_expected_schema_revision() + _expected_schema_revision_resolved = True + return _expected_schema_revision_cache + + @router.get("/capabilities") def get_acp_capabilities(app: App = Depends(App.get_instance)) -> dict: """Advertise the managed-agent contract and whether its schema is deployed. @@ -163,20 +267,21 @@ def get_acp_capabilities(app: App = Depends(App.get_instance)) -> dict: ``schema_ready=False``. """ revision = _schema_revision(app) - ready = revision == EXPECTED_SCHEMA_REVISION + expected = expected_schema_revision() + ready = revision is not None and revision == expected payload = { "managed_protocol_versions": [MANAGED_PROTOCOL_VERSION], "supported_runtimes": [MANAGED_RUNTIME], "schema_ready": ready, "schema_revision": revision, - "expected_schema_revision": EXPECTED_SCHEMA_REVISION, + "expected_schema_revision": expected, } if not ready: raise HTTPException( status_code=503, detail=( f"Managed agent schema not ready: have {revision!r}, " - f"want {EXPECTED_SCHEMA_REVISION!r}" + f"want {expected!r}" ), ) return payload @@ -224,7 +329,26 @@ def prepare_acp_grant( The plugin writes the returned grant where the agent process will find it. Grants are short-lived and single-use, so a stale handoff file is inert. + + The grant is the participant's funded launch credential, so it is gated by + the same live-enrollment/window/kill-switch rule as every other funded path. """ + user_id = _resolve_cookie_account(app, auth_token) + if user_id is not None: + db = app.get_db_session() + try: + kill_switch_check = operations_store.db_kill_switch_check(db) + access.require_live_enrollment( + db, account_id=user_id, kill_switch_check=kill_switch_check + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + finally: + db.close() + service = AcpAuthorizationService(app.get_redis_manager()) try: prepared = service.prepare_grant( @@ -253,6 +377,19 @@ def prepare_acp_grant( ) +def _resolve_cookie_account(app: App, auth_token: str) -> Optional[uuid.UUID]: + """Resolve the cookie's account id, or ``None`` when it is not authenticated.""" + if not auth_token: + return None + try: + info = app.get_redis_manager().get("auth_token", auth_token) + if not isinstance(info, dict) or not info.get("user_id"): + return None + return uuid.UUID(str(info["user_id"])) + except Exception: + return None + + @router.post("/session/exchange", response_model=ExchangeAcpGrantPostResponse) def exchange_acp_grant( request: Queries.ExchangeAcpGrant, @@ -391,8 +528,6 @@ def get_acp_agent_config( agent_profile = profile.name framework_version = profile.framework_version model = profile.model - base_url = profile.base_url - api_key_ref = profile.api_key_ref approval_policy = profile.approval_policy temperature = profile.temperature max_context_tokens = profile.max_context_tokens @@ -601,14 +736,26 @@ def get_participant_readiness( detail="No active study agent profile is assigned to this user", ) policy = _managed_policy(db, current_user.user_id, profile) - if profile.api_key_ref and not os.getenv(profile.api_key_ref, "").strip(): + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_profile( + db, profile + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + provider_module.resolve_upstream( + model=profile.model, + connection=connection, + framework_version=profile.framework_version, + ) + except provider_module.ProviderReadinessError as exc: raise HTTPException( status_code=503, - detail=( - "The assigned model provider is not configured on the study server " - f"(missing {profile.api_key_ref})" - ), - ) + detail={"code": exc.code, "message": exc.message}, + ) from exc return { "ready": True, "managed_protocol_version": MANAGED_PROTOCOL_VERSION, @@ -659,6 +806,8 @@ def create_managed_run( existing = crud.get_agent_task_by_external_run_id(db, body.run_id) if existing is not None: task = _require_managed_task(db, body, scope) + # A replayed run must still satisfy current funded state. + _require_funded_task(db, task) if not isinstance(task.policy_snapshot, dict): raise HTTPException(status_code=503, detail="Managed run policy is unavailable") return JSONResponse( @@ -685,6 +834,7 @@ def create_managed_run( framework_version=profile.framework_version, session_id=parent_session_id, owner_user_id=user_id, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), owner_project_id=project_id, external_run_id=body.run_id, agent_session_id=body.session_id, @@ -712,6 +862,8 @@ def create_managed_run( existing = crud.get_agent_task_by_external_run_id(db, body.run_id) if existing is not None: task = _require_managed_task(db, body, scope) + # A replayed run must still satisfy current funded state. + _require_funded_task(db, task) if not isinstance(task.policy_snapshot, dict): raise HTTPException(status_code=503, detail="Managed run policy is unavailable") return JSONResponse( @@ -740,12 +892,27 @@ async def run_managed_inference( db = app.get_db_session() try: task = _require_managed_task(db, body, scope) + # Live enrollment/window/kill-switch gate for funded managed inference. + _require_funded_task(db, task) policy = task.policy_snapshot if not _valid_managed_policy_snapshot(policy): raise HTTPException(status_code=503, detail="Managed run policy is unavailable") - profile = crud.get_agent_profile(db, task.agent_profile) + profile = None + if task.profile_id is not None: + profile = crud.get_agent_profile_by_id(db, task.profile_id) if profile is None: raise HTTPException(status_code=503, detail="Managed run provider configuration is unavailable") + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_task( + db, task + ) + try: + connection = provider_module.resolve_task_connection( + db, profile, funding_owner_user_id, owner_is_admin=owner_is_admin + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, detail={"code": exc.code, "message": exc.message} + ) from exc snapshot = { "task_id": task.task_id, "agent_profile": task.agent_profile, @@ -753,8 +920,7 @@ async def run_managed_inference( "temperature": policy.get("temperature"), "tools_json": json.dumps(policy.get("tools", [])), "framework_version": task.framework_version, - "base_url": profile.base_url, - "api_key_ref": profile.api_key_ref, + "connection": connection, "content_included": bool(policy.get("store_agent_content", False)), } finally: @@ -780,9 +946,8 @@ async def run_managed_inference( enrichment=None, agent_profile=snapshot["agent_profile"], model=snapshot["model"], + connection=snapshot["connection"], temperature=snapshot["temperature"], - base_url=snapshot["base_url"], - api_key_ref=snapshot["api_key_ref"], framework_version=snapshot["framework_version"], profile_tools_json=snapshot["tools_json"], content_included=snapshot["content_included"], diff --git a/src/backend/routers/agent/consent.py b/src/backend/routers/agent/consent.py index 92ed5bd..4ffa42c 100644 --- a/src/backend/routers/agent/consent.py +++ b/src/backend/routers/agent/consent.py @@ -27,9 +27,10 @@ import json import logging import uuid -from typing import Optional +from typing import TYPE_CHECKING, Optional -from sqlalchemy.orm import Session +if TYPE_CHECKING: + from sqlalchemy.orm import Session from database import crud from database.db_schemas import ( @@ -62,13 +63,18 @@ def _preference_allows_content(preference: Optional[str]) -> bool: def resolve_store_agent_content_for_user( - db: Session, user_id: Optional[uuid.UUID] + db: Session, + user_id: Optional[uuid.UUID], + *, + study_id: Optional[uuid.UUID] = None, ) -> bool: """Resolve the preference for a known user id. Returns False when the user can't be resolved at all: an unattributable request has nobody's consent to rely on, so it gets the conservative answer - even though the default for a real user is True. + even though the default for a real user is True. What is actually collected + follows the study's published telemetry policy, which is delivered to the + client and enforced by the privacy filter. """ if user_id is None: return False @@ -82,7 +88,12 @@ def resolve_store_agent_content_for_user( return _preference_allows_content(user.preference) -def resolve_store_agent_content(db: Session, session_id: uuid.UUID) -> bool: +def resolve_store_agent_content( + db: Session, + session_id: uuid.UUID, + *, + study_id: Optional[uuid.UUID] = None, +) -> bool: """Resolve the preference for the user owning ``session_id``. This is the plugin/proxy path, where the caller is identified by the @@ -95,10 +106,17 @@ def resolve_store_agent_content(db: Session, session_id: uuid.UUID) -> bool: return False if session is None or session.user_id is None: return False - return resolve_store_agent_content_for_user(db, session.user_id) + return resolve_store_agent_content_for_user( + db, session.user_id, study_id=study_id + ) -def resolve_store_agent_content_for_acp(db: Session, raw_user_id: str) -> bool: +def resolve_store_agent_content_for_acp( + db: Session, + raw_user_id: str, + *, + study_id: Optional[uuid.UUID] = None, +) -> bool: """Resolve the preference for an ACP-authorized agent process. ``raw_user_id`` comes from the server-derived ACP scope (see @@ -110,4 +128,4 @@ def resolve_store_agent_content_for_acp(db: Session, raw_user_id: str) -> bool: except (ValueError, TypeError, AttributeError): logging.warning(f"[Agent/consent] ACP scope user_id is not a UUID: {raw_user_id!r}") return False - return resolve_store_agent_content_for_user(db, user_uuid) + return resolve_store_agent_content_for_user(db, user_uuid, study_id=study_id) diff --git a/src/backend/routers/agent/ingest.py b/src/backend/routers/agent/ingest.py index 821a641..1b5a948 100644 --- a/src/backend/routers/agent/ingest.py +++ b/src/backend/routers/agent/ingest.py @@ -33,6 +33,7 @@ from backend.Responses import JsonResponseWithStatus from backend.routers.agent.acp_auth import require_acp_scope from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from backend.routers.research.access import resolve_research_binding from database import crud router = APIRouter() @@ -162,7 +163,12 @@ def _resolve_or_create_task( detail="No active agent profiles are configured on the server", ) profile = assignment.profile - content_included = resolve_store_agent_content_for_acp(db, scope.user_id) + content_included = resolve_store_agent_content_for_acp( + db, scope.user_id, study_id=assignment.study_id + ) + binding = resolve_research_binding( + db, account_id=owner_user_uuid, study_id=assignment.study_id + ) task = crud.create_agent_task( db, @@ -174,6 +180,7 @@ def _resolve_or_create_task( framework_version=profile.framework_version, source="code4me2_agent", owner_user_id=owner_user_uuid, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), owner_project_id=owner_project_uuid, external_run_id=run.run_id, agent_session_id=run.session_id, @@ -185,6 +192,12 @@ def _resolve_or_create_task( study_arm_name=assignment.arm_name, study_arm_is_baseline=assignment.is_baseline, consent_content_storage=content_included, + # Explicit phase-05 attribution: resolved from the authorized account + # and the frozen assignment, never guessed. A non-research task keeps + # these NULL (an explicit "no research context"). + research_session_id=binding.research_session_id if binding else None, + enrollment_id=binding.enrollment_id if binding else None, + study_revision_id=binding.study_revision_id if binding else None, ) logging.info( f"[Agent/ingest] created task {task.task_id} for run {run.run_id} " @@ -227,8 +240,13 @@ def ingest_agent_events( task, _created = _resolve_or_create_task(db, run=body.run, scope=scope) # Consent is resolved from the *server's* record of the user's - # preference, not from the `privacy` block in the payload. - content_included = resolve_store_agent_content_for_acp(db, scope.user_id) + # preference, not from the `privacy` block in the payload. The research + # enrollment gate is applied on top, so a non-ACTIVE research + # enrollment denies content collection even when the legacy preference + # would allow it. + content_included = resolve_store_agent_content_for_acp( + db, scope.user_id, study_id=task.study_id + ) ingested, skipped = ingest_module.ingest_event_batch( db, diff --git a/src/backend/routers/agent/profiles.py b/src/backend/routers/agent/profiles.py index 1c04dc9..282cf6a 100644 --- a/src/backend/routers/agent/profiles.py +++ b/src/backend/routers/agent/profiles.py @@ -1,33 +1,33 @@ -"""Admin CRUD for agent profiles, plus the plugin-facing registry. +"""Researcher-owned agent profiles plus the plugin-facing registry. Endpoints (mounted under ``/api/agent``): GET /available-tools tool names a profile may select - GET /profiles list profiles (admin) - POST /profiles create a profile (admin) - PUT /profiles/{id} update a profile (admin) - DELETE /profiles/{id} delete a profile (admin) + GET /profiles list the caller's profiles (admin: all) + POST /profiles create a profile + PUT /profiles/{id} update a profile + DELETE /profiles/{id} archive a profile GET /registry profiles in the shape the plugin consumes - GET /assignments list A/B assignments (admin) - PUT /assignments/{user_id} add a study-specific assignment (admin) - DELETE /assignments/{user_id} clear a user's assignments (admin) -Profiles are the unit of experimental control: a profile fixes the runtime, the -provider, the model, the tool allowlist and the approval policy, and users are -assigned to one as an A/B arm. +A profile is a private, editable template owned by exactly one researcher +(``agent_profile.owner_user_id``). It selects an administrator-managed +``provider_connection`` and a ``model`` from that connection's allowed list; the +release it pins is the approved artifact. The provider endpoint and secret live +on the connection, never on the profile. Names are unique per owner. + +There is no client-chosen arm: task/run assignment is server-authoritative (see +``agents.registry``). """ from __future__ import annotations import json -import re import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime from datetime import datetime from typing import Any, Optional -from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from sqlalchemy.exc import IntegrityError from agents.tools import KNOWN_AGENT_TOOLS, tools_for_framework @@ -38,41 +38,72 @@ get_current_user, require_admin, ) +from backend.routers.research.access import ( + is_owner, + require_owner, + require_researcher, +) from database import crud -from database.db_schemas import ( - AgentProfile, - AgentStudyAssignment, - Study, - StudyAgentProfile, - User, +from database.db_schemas import AgentProfile +from research.study.agents import store as registry_store +from research.study.agents.distributions import ( + distribution_supported_platforms, + resolve_distribution_view, ) router = APIRouter() -# Runtimes a profile may target. Anything else would mint tasks the plugin has -# no way to launch, so it's rejected at the API boundary rather than failing -# later at agent-startup time. SUPPORTED_FRAMEWORKS = ("code4me2-agent", "goose", "codex") SUPPORTED_APPROVAL_POLICIES = ("auto", "per_step", "suggestion_only") +def _models_for(connection: Any) -> list[str]: + try: + parsed = json.loads(getattr(connection, "models_json", None) or "[]") + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(parsed, list): + return [] + return [str(item) for item in parsed] + + class AgentProfilePayload(BaseModel): + """A profile template payload. + + ``extra="forbid"`` deliberately rejects the retired fields + (``base_url``/``api_key_ref``/``distribution_mode``/``agent_package``/ + ``agent_command``): researchers select an authorized ``connection_id`` and a + model, never an endpoint or a secret reference. + """ + + model_config = ConfigDict(extra="forbid") + name: str = Field(..., min_length=1) model: str = Field(..., min_length=1) framework_version: str = Field(default="code4me2-agent") - # Generic OpenAI-compatible endpoint. None = fall back to the server default. - base_url: Optional[str] = Field(default=None) - # Name of the env var holding the upstream key — never the key itself, so a - # profile is always safe to read back over the API. - api_key_ref: Optional[str] = Field(default=None) + # Administrator-managed provider connection this profile uses. + connection_id: uuid.UUID + # Exact approved artifact pin (agent_release.release_id). + release_id: Optional[str] = Field(default=None) tools_json: str = Field(default="[]") approval_policy: str = Field(..., min_length=1) max_steps: int = Field(..., ge=1) is_active: bool = Field(default=True) - # None = don't inject; let the provider use its own default. temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) max_context_tokens: Optional[int] = Field(default=None, ge=1) + @field_validator("release_id") + @classmethod + def normalize_release_id(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + candidate = value.strip() + if not candidate: + return None + if candidate.lower() == "latest": + raise ValueError("release_id must be an immutable pin, not 'latest'") + return candidate + @field_validator("name", "model") @classmethod def validate_required_text(cls, value: str) -> str: @@ -81,26 +112,6 @@ def validate_required_text(cls, value: str) -> str: raise ValueError("value must not be blank") return normalized - @field_validator("base_url") - @classmethod - def validate_base_url(cls, value: Optional[str]) -> Optional[str]: - if value is None or not value.strip(): - return None - candidate = value.strip().rstrip("/") - parsed = urlparse(candidate) - if ( - parsed.scheme not in {"http", "https"} - or not parsed.hostname - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - ): - raise ValueError( - "base_url must be an HTTP(S) endpoint without credentials, query, or fragment" - ) - return candidate - @field_validator("tools_json") @classmethod def validate_tools_json(cls, value: str) -> str: @@ -112,7 +123,6 @@ def validate_tools_json(cls, value: str) -> str: isinstance(item, str) for item in parsed ): raise ValueError("tools_json must be a JSON array of strings") - # Re-serialise so stored values are canonical regardless of input spacing. return json.dumps(parsed) @field_validator("framework_version") @@ -147,43 +157,93 @@ def validate_tools_for_framework(self): ) return self - @field_validator("api_key_ref") - @classmethod - def validate_api_key_ref(cls, value: Optional[str]) -> Optional[str]: - """Reject anything that looks like a key rather than a variable name. - - This is a guardrail against the obvious mistake: pasting the actual - secret into the field. Env var names are short and uppercase, so a long - value or one containing key-ish punctuation is almost certainly a - credential that must not be persisted. - """ - if value is None: - return None - candidate = value.strip() - if not candidate: - return None - if not re.fullmatch(r"[A-Z_][A-Z0-9_]{0,127}", candidate): - raise ValueError( - "api_key_ref must be the NAME of an environment variable " - "(e.g. OPENAI_API_KEY), not an API key value" - ) - return candidate + +def _authorize_connection( + db: Any, payload: AgentProfilePayload, current_user: AuthenticatedUser +) -> Any: + """Resolve the selected connection and enforce grant + model allowlist.""" + connection = crud.get_provider_connection(db, payload.connection_id) + if connection is None: + raise HTTPException( + status_code=422, + detail={ + "code": "CONNECTION_UNRESOLVED", + "field": "connection_id", + "message": "the selected provider connection does not exist", + }, + ) + if not current_user.is_admin and not crud.provider_connection_is_available( + db, connection.connection_id, current_user.user_id + ): + raise HTTPException( + status_code=403, + detail={ + "code": "CONNECTION_NOT_GRANTED", + "message": ( + "the caller is not authorized to use this provider connection" + ), + }, + ) + if payload.model not in _models_for(connection): + raise HTTPException( + status_code=422, + detail={ + "code": "MODEL_NOT_ALLOWED", + "field": "model", + "message": ( + f"model {payload.model!r} is not allowed by connection " + f"{connection.label!r}" + ), + }, + ) + return connection + + +def _release_for(db: Any, profile: AgentProfile): + """Rehydrate the registry release a profile pins, if any.""" + release_id = getattr(profile, "release_id", None) + if not release_id: + return None + row = registry_store.get_release(db, release_id) + if row is None: + return None + return registry_store.row_to_release(row) + + +def _connection_summary(db: Any, profile: AgentProfile) -> Optional[dict[str, Any]]: + if getattr(profile, "connection_id", None) is None: + return None + connection = crud.get_provider_connection(db, profile.connection_id) + if connection is None: + return {"connection_id": str(profile.connection_id), "label": None} + return { + "connection_id": str(connection.connection_id), + "label": connection.label, + "models": _models_for(connection), + "is_active": bool(connection.is_active), + } -def _profile_to_dict(profile: AgentProfile) -> dict[str, Any]: +def _profile_to_dict(db: Any, profile: AgentProfile) -> dict[str, Any]: + release = _release_for(db, profile) + view = resolve_distribution_view(profile, release) return { "profile_id": str(profile.profile_id), + "owner_user_id": str(profile.owner_user_id), "name": profile.name, "model": profile.model, "framework_version": profile.framework_version, - "base_url": profile.base_url, - "api_key_ref": profile.api_key_ref, "tools_json": profile.tools_json, "approval_policy": profile.approval_policy, "max_steps": profile.max_steps, "is_active": profile.is_active, "temperature": profile.temperature, "max_context_tokens": profile.max_context_tokens, + "connection": _connection_summary(db, profile), + "release_id": view.release_id, + "release_version": view.version, + "verified": view.verified, + "supported_platforms": distribution_supported_platforms(release), "created_at": ( profile.created_at.isoformat() if isinstance(profile.created_at, datetime) @@ -193,12 +253,7 @@ def _profile_to_dict(profile: AgentProfile) -> dict[str, Any]: def _profile_to_registry_dict(profile: AgentProfile) -> dict[str, Any]: - """Shape a profile for the plugin, including the env the runtime expects. - - Note that ``api_key_ref`` is exposed but the key is not: the plugin only - ever launches a *local* agent process, and the process reads the credential - from the backend at runtime, so the secret never transits this endpoint. - """ + """Shape a profile for the plugin: non-secret identity only.""" try: tools = json.loads(profile.tools_json or "[]") except json.JSONDecodeError: @@ -216,14 +271,13 @@ def _profile_to_registry_dict(profile: AgentProfile) -> dict[str, Any]: "name": profile.name, "model": profile.model, "framework_version": profile.framework_version, - "base_url": profile.base_url, - "api_key_ref": profile.api_key_ref, "tools": tools, "approval_policy": profile.approval_policy, "max_steps": profile.max_steps, "max_context_tokens": profile.max_context_tokens, "is_active": profile.is_active, "temperature": profile.temperature, + "release_id": getattr(profile, "release_id", None), "env": env, } @@ -235,8 +289,9 @@ def list_available_tools( description="Restrict to the tools relevant for one runtime " "(code4me2-agent | goose | codex). Omit for the full catalogue.", ), - current_user: AuthenticatedUser = Depends(require_admin), + current_user: AuthenticatedUser = Depends(get_current_user), ): + require_researcher(current_user) tools = ( tools_for_framework(framework_version) if framework_version @@ -254,15 +309,19 @@ def list_available_tools( @router.get("/profiles", summary="List agent profiles") def list_agent_profiles( - current_user: AuthenticatedUser = Depends(require_admin), + current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): + require_researcher(current_user) db = app.get_db_session() try: - profiles = crud.list_agent_profiles(db) + owner_scope = None if current_user.is_admin else current_user.user_id + profiles = crud.list_agent_profiles(db, owner_scope) return JsonResponseWithStatus( status_code=200, - content={"profiles": [_profile_to_dict(p) for p in profiles]}, + content={ + "profiles": [_profile_to_dict(db, profile) for profile in profiles] + }, ) finally: db.close() @@ -271,18 +330,21 @@ def list_agent_profiles( @router.post("/profiles", summary="Create an agent profile") def create_agent_profile( payload: AgentProfilePayload, - current_user: AuthenticatedUser = Depends(require_admin), + current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): + require_researcher(current_user) db = app.get_db_session() try: + _authorize_connection(db, payload, current_user) profile = crud.create_agent_profile( db, + owner_user_id=current_user.user_id, name=payload.name, model=payload.model, framework_version=payload.framework_version, - base_url=payload.base_url, - api_key_ref=payload.api_key_ref, + connection_id=payload.connection_id, + release_id=payload.release_id, tools_json=payload.tools_json, approval_policy=payload.approval_policy, max_steps=payload.max_steps, @@ -291,62 +353,83 @@ def create_agent_profile( max_context_tokens=payload.max_context_tokens, ) return JsonResponseWithStatus( - status_code=201, content={"profile": _profile_to_dict(profile)} + status_code=201, content={"profile": _profile_to_dict(db, profile)} ) except IntegrityError as exc: db.rollback() raise HTTPException( - status_code=409, detail="Agent profile name already exists" + status_code=409, + detail="You already have a profile with that name", ) from exc finally: db.close() +@router.get("/profiles/{profile_id}", summary="Fetch one agent profile") +def get_agent_profile( + profile_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + profile = crud.get_agent_profile_by_id(db, profile_id) + if profile is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, profile.owner_user_id, subject="profile") + return JsonResponseWithStatus( + status_code=200, content={"profile": _profile_to_dict(db, profile)} + ) + finally: + db.close() + + @router.put("/profiles/{profile_id}", summary="Update an agent profile") def update_agent_profile( profile_id: uuid.UUID, payload: AgentProfilePayload, - current_user: AuthenticatedUser = Depends(require_admin), + current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): - """Update a profile. + """Update a profile template. - A profile used by an active study cannot change. Agent tasks also copy the - effective runtime configuration at creation, so later changes never alter - completed task records. + Editing a template never rewrites a published study snapshot: publication + frozen the profile's configuration into the revision. """ + require_researcher(current_user) db = app.get_db_session() try: + existing = crud.get_agent_profile_by_id(db, profile_id) + if existing is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, existing.owner_user_id, subject="profile") + _authorize_connection(db, payload, current_user) profile = crud.update_agent_profile( db, profile_id=profile_id, name=payload.name, model=payload.model, framework_version=payload.framework_version, - base_url=payload.base_url, - api_key_ref=payload.api_key_ref, tools_json=payload.tools_json, approval_policy=payload.approval_policy, max_steps=payload.max_steps, is_active=payload.is_active, temperature=payload.temperature, max_context_tokens=payload.max_context_tokens, + connection_id=payload.connection_id, + release_id=payload.release_id, + update_connection_id=True, + update_release_id=True, ) - if profile is None: - raise HTTPException(status_code=404, detail="Agent profile not found") return JsonResponseWithStatus( - status_code=200, content={"profile": _profile_to_dict(profile)} + status_code=200, content={"profile": _profile_to_dict(db, profile)} ) except IntegrityError as exc: - db.rollback() - raise HTTPException( - status_code=409, detail="Agent profile name already exists" - ) from exc - except crud.AgentProfileInActiveStudyError as exc: db.rollback() raise HTTPException( status_code=409, - detail="Profiles attached to an active study cannot be changed", + detail="You already have a profile with that name", ) from exc finally: db.close() @@ -355,25 +438,22 @@ def update_agent_profile( @router.delete("/profiles/{profile_id}", summary="Retire an agent profile") def delete_agent_profile( profile_id: uuid.UUID, - current_user: AuthenticatedUser = Depends(require_admin), + current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): - """Retire a profile that is not part of an active study.""" + """Archive a profile; frozen study snapshots are never erased.""" + require_researcher(current_user) db = app.get_db_session() try: - deleted = crud.delete_agent_profile(db, profile_id) - if not deleted: + existing = crud.get_agent_profile_by_id(db, profile_id) + if existing is None: raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, existing.owner_user_id, subject="profile") + crud.delete_agent_profile(db, profile_id) return JsonResponseWithStatus( status_code=200, content={"retired": True, "profile_id": str(profile_id)}, ) - except crud.AgentProfileInActiveStudyError as exc: - db.rollback() - raise HTTPException( - status_code=409, - detail="Profiles attached to an active study cannot be retired", - ) from exc finally: db.close() @@ -383,171 +463,14 @@ def list_agent_registry( current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): + require_researcher(current_user) db = app.get_db_session() try: - profiles = crud.list_agent_profiles(db) + owner_scope = None if current_user.is_admin else current_user.user_id + profiles = crud.list_agent_profiles(db, owner_scope) return JsonResponseWithStatus( status_code=200, content={"profiles": [_profile_to_registry_dict(p) for p in profiles]}, ) finally: db.close() - - -# ── A/B assignments ───────────────────────────────────────────────────────── -# -# Assignments are normally created automatically (sticky random draw) on a -# user's first agent task. These admin endpoints let you inspect them and pin a -# specific user to a specific profile (source="manual"), which overrides the -# random draw and is excluded from A/B analysis. - - -class AssignmentPayload(BaseModel): - profile_id: uuid.UUID - study_id: Optional[uuid.UUID] = None - - -def _assignment_to_dict(assignment: AgentStudyAssignment) -> dict[str, Any]: - profile = assignment.profile - return { - "assignment_id": str(assignment.assignment_id), - "study_id": str(assignment.study_id), - "user_id": str(assignment.user_id), - "profile_id": str(assignment.profile_id), - "profile_name": profile.name if profile is not None else None, - "arm_name": assignment.arm_name, - "is_baseline": assignment.is_baseline, - "source": assignment.source, - "assigned_at": ( - assignment.assigned_at.isoformat() - if isinstance(assignment.assigned_at, datetime) - else assignment.assigned_at - ), - } - - -@router.get("/assignments", summary="List all agent profile assignments") -def list_agent_assignments( - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - assignments = crud.list_agent_study_assignments(db) - return JsonResponseWithStatus( - status_code=200, - content={"assignments": [_assignment_to_dict(a) for a in assignments]}, - ) - finally: - db.close() - - -@router.get("/assignment-options", summary="List users and studies for agent assignments") -def list_agent_assignment_options( - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - users = db.query(User).order_by(User.name, User.email).all() - studies = db.query(Study).order_by(Study.is_active.desc(), Study.name).all() - profile_ids_by_study: dict[uuid.UUID, list[str]] = {} - for link in db.query(StudyAgentProfile).all(): - profile_ids_by_study.setdefault(link.study_id, []).append( - str(link.profile_id) - ) - return JsonResponseWithStatus( - status_code=200, - content={ - "users": [ - {"user_id": str(user.user_id), "name": user.name, "email": user.email} - for user in users - ], - "studies": [ - { - "study_id": str(study.study_id), - "name": study.name, - "is_active": study.is_active, - "profile_ids": profile_ids_by_study.get(study.study_id, []), - } - for study in studies - ], - }, - ) - finally: - db.close() - - -@router.put( - "/assignments/{user_id}", summary="Add an agent study assignment" -) -def set_agent_assignment( - user_id: uuid.UUID, - payload: AssignmentPayload, - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - study_id = payload.study_id - if study_id is None: - active_study = crud.get_active_agent_study(db) - if active_study is None: - raise HTTPException( - status_code=409, - detail="No active study with agent arms; specify an active study first", - ) - study_id = active_study.study_id - arm = crud.get_study_agent_profile(db, study_id, payload.profile_id) - if arm is None: - raise HTTPException( - status_code=400, - detail="Profile is not an arm of the specified study", - ) - profile = crud.get_agent_profile_by_id(db, payload.profile_id) - if profile is None: - raise HTTPException(status_code=404, detail="Agent profile not found") - if crud.get_agent_study_assignment(db, study_id, user_id) is not None: - raise HTTPException( - status_code=409, - detail="Study assignment already exists and cannot be changed", - ) - assignment = crud.create_agent_study_assignment( - db, - study_id=study_id, - user_id=user_id, - profile_id=profile.profile_id, - arm_name=profile.name, - is_baseline=arm.is_baseline, - source="manual", - ) - if assignment is None: - raise HTTPException( - status_code=409, - detail="Study assignment was created concurrently; reload assignments", - ) - return JsonResponseWithStatus( - status_code=201, content={"assignment": _assignment_to_dict(assignment)} - ) - finally: - db.close() - - -@router.delete("/assignments/{user_id}", summary="Clear a user's agent assignments") -def delete_agent_assignment( - user_id: uuid.UUID, - study_id: Optional[uuid.UUID] = Query(default=None), - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - deleted = crud.delete_agent_study_assignments(db, user_id, study_id) - if not deleted: - raise HTTPException(status_code=404, detail="Assignment not found") - return JsonResponseWithStatus( - status_code=200, - content={"deleted": True, "user_id": str(user_id)}, - ) - finally: - db.close() diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py index 729e73a..6cea18e 100644 --- a/src/backend/routers/agents.py +++ b/src/backend/routers/agents.py @@ -16,6 +16,7 @@ import logging import uuid +from datetime import datetime, timezone from typing import Any, Optional from fastapi import APIRouter, Cookie, Depends, HTTPException, Response @@ -23,13 +24,45 @@ from pydantic import BaseModel, Field from agents import inference, lifecycle, registry +from agents import provider as provider_module from App import App from backend.routers.agent.consent import resolve_store_agent_content +from backend.routers.research import access from database import crud +from research.analysis.operations import store as operations_store +from research.telemetry.adapters import LegacyFact, record_legacy_facts +from research.telemetry.enums import CoverageState +from research.telemetry.models import Coverage, EventMetrics router = APIRouter(tags=["Agent"]) +def _require_funded_task(db, task) -> None: + """Re-check live enrollment/window/kill switch for a study-funded task. + + Non-research operational tasks (no ``study_id``) are unaffected. A research + task must resolve an ACTIVE enrollment in its own study, with the study + window open and no operator kill switch engaged. + """ + if getattr(task, "study_id", None) is None: + return + kill_switch_check = operations_store.db_kill_switch_check( + db, study_id=task.study_id, revision_id=None, enrollment_id=None + ) + try: + access.require_live_enrollment( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + kill_switch_check=kill_switch_check, + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + + def require_session( app: App = Depends(App.get_instance), session_token: str = Cookie(""), @@ -143,8 +176,17 @@ def create_agent_task( profile = assignment.profile # task_description is the user's own words — content, so honour the - # consent gate even at creation time. - content_included = resolve_store_agent_content(db, session_id) + # consent gate even at creation time. The research enrollment gate is + # applied on top of the legacy preference. + content_included = resolve_store_agent_content( + db, session_id, study_id=assignment.study_id + ) + # Explicit phase-05 attribution: resolved from the authorized account and + # the frozen assignment, never guessed (NULL when there is no live + # research context). + binding = access.resolve_research_binding( + db, account_id=session.user_id, study_id=assignment.study_id + ) task = crud.create_agent_task( db, @@ -157,6 +199,7 @@ def create_agent_task( task_description=body.task_description if content_included else None, session_id=session_id, owner_user_id=session.user_id, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), task_id=body.task_id, source="plugin", study_id=assignment.study_id, @@ -165,6 +208,9 @@ def create_agent_task( study_arm_name=assignment.arm_name, study_arm_is_baseline=assignment.is_baseline, consent_content_storage=content_included, + research_session_id=binding.research_session_id if binding else None, + enrollment_id=binding.enrollment_id if binding else None, + study_revision_id=binding.study_revision_id if binding else None, ) logging.info( f"[Agent/task] created task_id={task.task_id} profile={profile.name!r} " @@ -228,12 +274,36 @@ async def run_agent_inference( db = app.get_db_session() try: task = _require_own_task(db, body.task_id, session_id) + # Funded research use requires a live enrollment and an open study + # window, re-checked here on every call (a previously issued capability + # never substitutes for current server state). + _require_funded_task(db, task) # Resolved server-side from the stored preference, never from the - # request body — see backend.routers.agent.consent. - content_included = resolve_store_agent_content(db, session_id) - profile = task.profile or crud.get_agent_profile(db, task.agent_profile) - base_url = profile.base_url if profile else None - api_key_ref = profile.api_key_ref if profile else None + # request body — see backend.routers.agent.consent. The research + # enrollment gate is applied on top. + content_included = resolve_store_agent_content( + db, session_id, study_id=task.study_id + ) + profile = task.profile + if profile is None and task.profile_id is not None: + profile = crud.get_agent_profile_by_id(db, task.profile_id) + # The connection grant belongs to the study's funding owner, never the + # participant (whose own admin flag must not widen access). + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_task( + db, task + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, + detail={"code": exc.code, "message": exc.message}, + ) from exc task_snapshot = { "agent_profile": task.agent_profile, "model": task.model, @@ -257,9 +327,8 @@ async def run_agent_inference( enrichment=body.enrichment, agent_profile=task_snapshot["agent_profile"], model=task_snapshot["model"], + connection=connection, temperature=task_snapshot["temperature"], - base_url=base_url, - api_key_ref=api_key_ref, framework_version=task_snapshot["framework_version"], profile_tools_json=task_snapshot["tools_json"], content_included=content_included, @@ -278,6 +347,46 @@ async def run_agent_inference( } +def _span_fact(span: "SpanPayload", event_type: str) -> "LegacyFact": + """Build the canonical fact for one OTel child span (structural only).""" + attrs = span.attributes or {} + payload: dict[str, Any] = {} + model = attrs.get("llm.model") or attrs.get("model") + if model: + payload["model"] = model + if event_type == "tool_call" and attrs.get("tool.name"): + payload["tool_name"] = attrs.get("tool.name") + prompt = _safe_int(attrs.get("llm.prompt_tokens")) + completion = _safe_int(attrs.get("llm.completion_tokens")) + total = _safe_int(attrs.get("llm.total_tokens")) + counts: dict[str, int] = {} + if prompt is not None: + counts["prompt_tokens"] = prompt + if completion is not None: + counts["completion_tokens"] = completion + return LegacyFact( + kind=event_type, + occurred_at=datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics( + usage_tokens=total, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability="usage", + ), + latency_ms=span.duration_ms if span.duration_ms > 0 else None, + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="span"), + source_event_id=span.span_id, + emitter_id="proxy", + ) + + def _safe_int(value: Any) -> Optional[int]: """Coerce an OTel attribute value to int, or None if it isn't numeric.""" try: @@ -368,6 +477,20 @@ def upload_agent_telemetry( f"steps={len(child_spans)} tokens={total_prompt}/{total_completion}" ) + task_row = crud.get_agent_task(db, task_id) + span_facts = [ + _span_fact(span, _SPAN_EVENT_TYPE.get(span.name, "observation")) + for span in child_spans + ] + if task_row is not None and record_legacy_facts( + db, task=task_row, facts=span_facts + ): + logging.info( + f"[Agent/telemetry] canonicalized {len(child_spans)} span(s) " + f"for task {task_id}" + ) + return Response(status_code=204) + base_index = ( crud.reserve_agent_event_indexes(db, task_id, len(child_spans)) if child_spans diff --git a/src/backend/routers/analytics/agents.py b/src/backend/routers/analytics/agents.py index bb10b70..022eae7 100644 --- a/src/backend/routers/analytics/agents.py +++ b/src/backend/routers/analytics/agents.py @@ -1,27 +1,25 @@ -"""Privacy-scoped aggregate analytics for agent invocations.""" +"""Privacy-scoped agent analytics over the canonical event authority (phase 06). -from datetime import datetime, timedelta, timezone +Both routes delegate to one owner-scoped canonical query path +(:mod:`research.analysis.read_models.dashboard`), which reads ``research_event`` +joined to ``agent_task`` by the explicit phase-05 binding +(``research_event.agent_run_id = agent_task.external_run_id``). Ownership is +enforced before the joins; there is no ACP-session-id or time-window bridging, +and a legacy run with no canonical events renders explicit unavailable values. +""" + +from datetime import datetime, timezone from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import text from App import App from backend.Responses import JsonResponseWithStatus +from research.analysis.read_models.dashboard import agent_overview, agent_run_detail + from .auth_utils import AuthenticatedUser, get_current_user router = APIRouter() -_TIME_WINDOWS = {"7d": 7, "30d": 30, "90d": 90} - - -def _as_number(value, integer: bool = False): - if value is None: - return 0 if integer else 0.0 - return int(value) if integer else float(value) - - -def _iso(value): - return value.isoformat() if value is not None else None @router.get("/overview") @@ -34,335 +32,28 @@ def get_agent_overview( current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): - """Aggregate task/event telemetry without returning content payloads.""" - days = _TIME_WINDOWS.get(time_window) - if days is None: - raise HTTPException(status_code=400, detail="time_window must be 7d, 30d, or 90d") - + """Canonical, owner-scoped agent telemetry without content payloads.""" db = app.get_db_session() try: - end_time = datetime.now(timezone.utc) - start_time = end_time - timedelta(days=days) - params = {"start_time": start_time, "end_time": end_time} - conditions = ["t.created_at BETWEEN :start_time AND :end_time"] - - if current_user.is_admin and user_id: - params["user_id"] = user_id - conditions.append("CAST(t.owner_user_id AS TEXT) = :user_id") - elif not current_user.is_admin: - params["user_id"] = str(current_user.user_id) - conditions.append("t.owner_user_id = CAST(:user_id AS UUID)") - - if framework: - params["framework"] = framework - conditions.append("t.framework_version = :framework") - if model: - params["model"] = model - conditions.append("t.model = :model") - if profile: - params["profile"] = profile - conditions.append("t.agent_profile = :profile") - - where = " AND ".join(conditions) - - task_query = f""" - SELECT - COUNT(*) AS total_tasks, - COUNT(*) FILTER (WHERE t.status = 'done') AS completed_tasks, - COUNT(*) FILTER (WHERE t.status = 'failed') AS failed_tasks, - COUNT(*) FILTER (WHERE t.status IN ('pending', 'running')) AS open_tasks, - COALESCE(AVG(t.total_steps), 0) AS avg_steps, - COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) - FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) - AS avg_task_duration_ms, - COALESCE(SUM(t.input_tokens), 0) AS input_tokens, - COALESCE(SUM(t.output_tokens), 0) AS output_tokens - FROM agent_task t - WHERE {where} - """ - task = db.execute(text(task_query), params).one() - - events_query = f""" - SELECT - COUNT(*) FILTER (WHERE e.event_type = 'model_request') AS model_requests, - COUNT(*) FILTER (WHERE e.event_type = 'model_call') AS model_calls, - COUNT(*) FILTER (WHERE e.event_type = 'tool_request') AS tool_requests, - COUNT(*) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, - COUNT(*) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures, - COALESCE(AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call'), 0) - AS avg_model_latency_ms, - COALESCE(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY e.latency_ms) - FILTER (WHERE e.event_type = 'model_call'), 0) - AS p95_model_latency_ms, - COALESCE(SUM(e.prompt_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) - AS provider_input_tokens, - COALESCE(SUM(e.completion_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) - AS model_output_tokens, - COUNT(*) FILTER (WHERE e.event_type = 'model_call' AND e.upstream_status = 429) - AS rate_limit_retries, - COUNT(*) FILTER (WHERE e.event_type = 'model_call' - AND e.upstream_status >= 200 AND e.upstream_status < 400) - AS successful_model_calls, - COALESCE(SUM( - CASE WHEN e.event_type = 'model_call' - THEN COALESCE((NULLIF(e.extra_json, '')::jsonb ->> 'tool_schema_bytes')::bigint, 0) - ELSE 0 END - ), 0) AS tool_schema_bytes, - COALESCE(SUM(e.context_window_size_bytes) - FILTER (WHERE e.event_type = 'model_call'), 0) - AS conversation_context_bytes, - COALESCE(SUM(e.tool_result_length) - FILTER (WHERE e.event_type = 'tool_call'), 0) - AS tool_result_bytes, - COALESCE(SUM(e.total_tokens) FILTER (WHERE e.event_type = 'model_call'), 0) - AS event_tokens - FROM agent_event e - JOIN agent_task t ON t.task_id = e.task_id - WHERE {where} - """ - events = db.execute(text(events_query), params).one() - - edits_query = f""" - SELECT - COUNT(ed.edit_id) AS total_edits, - COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS TRUE) AS accepted_edits, - COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS FALSE) AS rejected_edits - FROM agent_task t - LEFT JOIN agent_edit ed ON ed.task_id = t.task_id - WHERE {where} - """ - edits = db.execute(text(edits_query), params).one() - - profiles_query = f""" - SELECT - COALESCE(t.agent_profile, 'unknown') AS profile_name, - COALESCE(t.framework_version, 'unknown') AS framework_version, - COALESCE(t.model, 'unknown') AS model, - COUNT(*) AS tasks, - COUNT(*) FILTER (WHERE t.status = 'done') AS completed, - COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, - COALESCE(AVG(t.total_steps), 0) AS avg_steps - FROM agent_task t - WHERE {where} - GROUP BY 1, 2, 3 - ORDER BY tasks DESC, profile_name - """ - profile_rows = db.execute(text(profiles_query), params).fetchall() - - tools_query = f""" - SELECT - COALESCE(e.tool_name, 'unknown') AS tool_name, - COUNT(*) FILTER (WHERE e.event_type = 'tool_call') AS calls, - COUNT(*) FILTER (WHERE e.event_type = 'tool_failed') AS failures, - COALESCE(AVG(e.latency_ms), 0) AS avg_latency_ms - FROM agent_event e - JOIN agent_task t ON t.task_id = e.task_id - WHERE {where} AND e.event_type IN ('tool_call', 'tool_failed') - GROUP BY 1 - ORDER BY calls DESC, tool_name - LIMIT 20 - """ - tool_rows = db.execute(text(tools_query), params).fetchall() - - runs_query = f""" - SELECT - t.task_id, - t.agent_profile, - t.framework_version, - t.model, - t.status, - t.total_steps, - t.created_at, - t.started_at, - t.completed_at, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, - COUNT(e.event_id) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures - FROM agent_task t - LEFT JOIN agent_event e ON e.task_id = t.task_id - WHERE {where} - GROUP BY t.task_id - ORDER BY t.created_at DESC - LIMIT 25 - """ - run_rows = db.execute(text(runs_query), params).fetchall() - - trend_query = f""" - SELECT - DATE_TRUNC('day', t.created_at) AS day, - COUNT(*) AS runs, - COUNT(*) FILTER (WHERE t.status = 'done') AS completed, - COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, - COALESCE( - AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) - FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), - 0 - ) AS avg_duration_ms - FROM agent_task t - WHERE {where} - GROUP BY 1 - ORDER BY 1 - """ - trend_rows = db.execute(text(trend_query), params).fetchall() - - latency_query = f""" - SELECT - CASE - WHEN e.latency_ms < 250 THEN '<250ms' - WHEN e.latency_ms < 500 THEN '250-499ms' - WHEN e.latency_ms < 1000 THEN '500-999ms' - WHEN e.latency_ms < 2000 THEN '1-2s' - ELSE '2s+' - END AS bucket, - CASE - WHEN e.latency_ms < 250 THEN 1 - WHEN e.latency_ms < 500 THEN 2 - WHEN e.latency_ms < 1000 THEN 3 - WHEN e.latency_ms < 2000 THEN 4 - ELSE 5 - END AS bucket_order, - COUNT(*) AS calls - FROM agent_event e - JOIN agent_task t ON t.task_id = e.task_id - WHERE {where} AND e.event_type = 'model_call' AND e.latency_ms IS NOT NULL - GROUP BY 1, 2 - ORDER BY bucket_order - """ - latency_rows = db.execute(text(latency_query), params).fetchall() - - event_types_query = f""" - SELECT e.event_type, COUNT(*) AS events - FROM agent_event e - JOIN agent_task t ON t.task_id = e.task_id - WHERE {where} - GROUP BY e.event_type - ORDER BY events DESC, e.event_type - """ - event_type_rows = db.execute(text(event_types_query), params).fetchall() - - errors_query = f""" - SELECT - CASE - WHEN e.upstream_status IS NOT NULL AND e.upstream_status >= 400 - THEN 'upstream_' || e.upstream_status::text - ELSE e.event_type - END AS reason, - COUNT(*) AS events - FROM agent_event e - JOIN agent_task t ON t.task_id = e.task_id - WHERE {where} - AND (e.event_type IN ('error', 'tool_failed', 'tool_denied') - OR e.upstream_status >= 400) - GROUP BY 1 - ORDER BY events DESC, reason - """ - error_rows = db.execute(text(errors_query), params).fetchall() - - total_tasks = _as_number(task.total_tasks, integer=True) - decided_edits = _as_number(edits.accepted_edits, integer=True) + _as_number( - edits.rejected_edits, integer=True - ) - return JsonResponseWithStatus( - status_code=200, - content={ - "filters": {"time_window": time_window, "framework": framework, "model": model, "profile": profile}, - "summary": { - "total_tasks": total_tasks, - "completed_tasks": _as_number(task.completed_tasks, integer=True), - "failed_tasks": _as_number(task.failed_tasks, integer=True), - "open_tasks": _as_number(task.open_tasks, integer=True), - "completion_rate": _as_number(task.completed_tasks, integer=True) / max(total_tasks, 1), - "avg_steps": _as_number(task.avg_steps), - "avg_task_duration_ms": _as_number(task.avg_task_duration_ms), - "input_tokens": _as_number(task.input_tokens, integer=True), - "output_tokens": _as_number(task.output_tokens, integer=True), - "model_requests": _as_number(events.model_requests, integer=True), - "model_calls": _as_number(events.model_calls, integer=True), - "tool_requests": _as_number(events.tool_requests, integer=True), - "tool_calls": _as_number(events.tool_calls, integer=True), - "failures": _as_number(events.failures, integer=True), - "avg_model_latency_ms": _as_number(events.avg_model_latency_ms), - "p95_model_latency_ms": _as_number(events.p95_model_latency_ms), - "provider_input_tokens": _as_number(events.provider_input_tokens, integer=True), - "model_output_tokens": _as_number(events.model_output_tokens, integer=True), - "successful_model_calls": _as_number(events.successful_model_calls, integer=True), - "rate_limit_retries": _as_number(events.rate_limit_retries, integer=True), - # These are estimates based on UTF-8 bytes / 4, not provider billing tokens. - "tool_schema_tokens_estimated": _as_number(events.tool_schema_bytes, integer=True) // 4, - "conversation_tokens_estimated": _as_number(events.conversation_context_bytes, integer=True) // 4, - "tool_result_tokens_estimated": _as_number(events.tool_result_bytes, integer=True) // 4, - "event_tokens": _as_number(events.event_tokens, integer=True), - "total_edits": _as_number(edits.total_edits, integer=True), - "edit_acceptance_rate": _as_number(edits.accepted_edits, integer=True) / max(decided_edits, 1), - }, - "profiles": [ - { - "profile_name": row.profile_name, - "framework_version": row.framework_version, - "model": row.model, - "tasks": _as_number(row.tasks, integer=True), - "completed": _as_number(row.completed, integer=True), - "failed": _as_number(row.failed, integer=True), - "completion_rate": _as_number(row.completed, integer=True) / max(_as_number(row.tasks, integer=True), 1), - "avg_steps": _as_number(row.avg_steps), - } - for row in profile_rows - ], - "tools": [ - { - "tool_name": row.tool_name, - "calls": _as_number(row.calls, integer=True), - "failures": _as_number(row.failures, integer=True), - "avg_latency_ms": _as_number(row.avg_latency_ms), - } - for row in tool_rows - ], - "recent_runs": [ - { - "task_id": str(row.task_id), - "profile_name": row.agent_profile, - "framework_version": row.framework_version, - "model": row.model, - "status": row.status, - "steps": _as_number(row.total_steps, integer=True), - "created_at": _iso(row.created_at), - "started_at": _iso(row.started_at), - "completed_at": _iso(row.completed_at), - "model_calls": _as_number(row.model_calls, integer=True), - "tool_calls": _as_number(row.tool_calls, integer=True), - "failures": _as_number(row.failures, integer=True), - } - for row in run_rows - ], - "trend": [ - { - "day": _iso(row.day), - "runs": _as_number(row.runs, integer=True), - "completed": _as_number(row.completed, integer=True), - "failed": _as_number(row.failed, integer=True), - "avg_duration_ms": _as_number(row.avg_duration_ms), - } - for row in trend_rows - ], - "latency_distribution": [ - {"bucket": row.bucket, "calls": _as_number(row.calls, integer=True)} - for row in latency_rows - ], - "event_types": [ - {"event_type": row.event_type, "events": _as_number(row.events, integer=True)} - for row in event_type_rows - ], - "errors": [ - {"reason": row.reason, "events": _as_number(row.events, integer=True)} - for row in error_rows - ], - }, + content = agent_overview( + db, + current_user, + time_window=time_window, + framework=framework, + model=model, + profile=profile, + user_id=user_id, ) + return JsonResponseWithStatus(status_code=200, content=content) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error except HTTPException: raise except Exception as error: db.rollback() - raise HTTPException(status_code=500, detail=f"Error retrieving agent analytics: {error}") from error + raise HTTPException( + status_code=500, detail=f"Error retrieving agent analytics: {error}" + ) from error finally: db.close() @@ -373,77 +64,19 @@ def get_agent_run_detail( current_user: AuthenticatedUser = Depends(get_current_user), app: App = Depends(App.get_instance), ): - """Return structural event details for one authorized run, never content.""" + """Return one authorized run's canonical event timeline, never content.""" db = app.get_db_session() try: - params = {"task_id": task_id} - owner_filter = "" - if not current_user.is_admin: - params["user_id"] = str(current_user.user_id) - owner_filter = "AND CAST(t.owner_user_id AS TEXT) = :user_id" - - task_query = f""" - SELECT t.task_id, t.agent_profile, t.framework_version, t.model, - t.status, t.total_steps, t.created_at, t.started_at, t.completed_at - FROM agent_task t - WHERE CAST(t.task_id AS TEXT) = :task_id {owner_filter} - """ - task = db.execute(text(task_query), params).one_or_none() - if task is None: + content = agent_run_detail(db, current_user, task_id=task_id) + if content is None: raise HTTPException(status_code=404, detail="Agent run not found") - - events_query = """ - SELECT event_index, event_type, source, latency_ms, created_at, - occurred_at, span_id, parent_span_id, model, tool_name, - prompt_tokens, completion_tokens, total_tokens, - finish_reason, upstream_status, step_index - FROM agent_event - WHERE task_id = CAST(:task_id AS UUID) - ORDER BY event_index - """ - events = db.execute(text(events_query), {"task_id": task_id}).fetchall() - - return JsonResponseWithStatus( - status_code=200, - content={ - "run": { - "task_id": str(task.task_id), - "profile_name": task.agent_profile, - "framework_version": task.framework_version, - "model": task.model, - "status": task.status, - "steps": _as_number(task.total_steps, integer=True), - "created_at": _iso(task.created_at), - "started_at": _iso(task.started_at), - "completed_at": _iso(task.completed_at), - }, - "events": [ - { - "event_index": _as_number(row.event_index, integer=True), - "event_type": row.event_type, - "source": row.source, - "latency_ms": _as_number(row.latency_ms), - "created_at": _iso(row.created_at), - "occurred_at": _iso(row.occurred_at), - "span_id": row.span_id, - "parent_span_id": str(row.parent_span_id) if row.parent_span_id else None, - "model": row.model, - "tool_name": row.tool_name, - "prompt_tokens": _as_number(row.prompt_tokens, integer=True), - "completion_tokens": _as_number(row.completion_tokens, integer=True), - "total_tokens": _as_number(row.total_tokens, integer=True), - "finish_reason": row.finish_reason, - "upstream_status": row.upstream_status, - "step_index": row.step_index, - } - for row in events - ], - }, - ) + return JsonResponseWithStatus(status_code=200, content=content) except HTTPException: raise except Exception as error: db.rollback() - raise HTTPException(status_code=500, detail=f"Error retrieving agent run: {error}") from error + raise HTTPException( + status_code=500, detail=f"Error retrieving agent run: {error}" + ) from error finally: - db.close() \ No newline at end of file + db.close() diff --git a/src/backend/routers/analytics/auth_utils.py b/src/backend/routers/analytics/auth_utils.py index 3a0f800..16bbed8 100644 --- a/src/backend/routers/analytics/auth_utils.py +++ b/src/backend/routers/analytics/auth_utils.py @@ -21,12 +21,22 @@ class AuthenticatedUser: """Container for authenticated user information.""" - - def __init__(self, user_id: uuid.UUID, is_admin: bool, email: str, name: str): + + def __init__( + self, + user_id: uuid.UUID, + is_admin: bool, + email: str, + name: str, + can_research: bool = False, + ): self.user_id = user_id self.is_admin = is_admin self.email = email self.name = name + # Administrator-enabled researcher account (P3). There is no per-study + # role table; this single flag is the researcher authority. + self.can_research = can_research def get_current_user( @@ -69,7 +79,8 @@ def get_current_user( user_id=user.user_id, is_admin=user.is_admin, email=user.email, - name=user.name + name=user.name, + can_research=bool(getattr(user, "can_research", False)), ) except Exception as e: diff --git a/src/backend/routers/analytics/studies.py b/src/backend/routers/analytics/studies.py index ca9670f..9903c0d 100644 --- a/src/backend/routers/analytics/studies.py +++ b/src/backend/routers/analytics/studies.py @@ -94,33 +94,27 @@ def create_study( # Validate the optional agent-profile arms. Done before any writes so a # bad arm list can't leave a half-configured study behind. - agent_profile_uuids: List[uuid.UUID] = [] - baseline_profile_uuid: Optional[uuid.UUID] = None + # + # The legacy ``study_agent_profile`` arm authority was removed by the + # wave-1 consolidation (profiles are researcher-owned templates, and + # agent allocation is enrollment-scoped via the research + # ``study_assignment`` authority). Completion studies keep working, but + # they can no longer attach agent-profile arms through this route. try: agent_profile_uuids = [ uuid.UUID(pid) for pid in study_request.agent_profile_ids ] - if study_request.baseline_agent_profile_id is not None: - baseline_profile_uuid = uuid.UUID( - study_request.baseline_agent_profile_id - ) except (ValueError, TypeError): raise HTTPException(status_code=400, detail="Invalid agent profile ID format") - - for profile_uuid in agent_profile_uuids: - if crud.get_agent_profile_by_id(db_session, profile_uuid) is None: - raise HTTPException( - status_code=400, - detail=f"Agent profile {profile_uuid} not found", - ) - if ( - baseline_profile_uuid is not None - and baseline_profile_uuid not in agent_profile_uuids - ): + if agent_profile_uuids or study_request.baseline_agent_profile_id is not None: raise HTTPException( status_code=400, - detail="Baseline agent profile must be one of the selected profiles", + detail=( + "agent-profile arms are no longer supported on completion " + "studies; create a research study for agent conditions" + ), ) + baseline_profile_uuid: Optional[uuid.UUID] = None # Check if there's already an active study active_study_query = """ @@ -161,20 +155,6 @@ def create_study( if is_active: assign_users_to_study(db_session, study_id, study_request.config_ids) - # Attach the agent-profile arms. Persisted regardless of active state: - # only an *active* study's arms are drawn from - # (crud.list_active_study_agent_profiles), and agent assignment is - # sticky-lazy — drawn on a user's first agent task rather than - # pre-assigned like completion configs — so there's nothing to backfill - # here. - if agent_profile_uuids: - crud.set_study_agent_profiles( - db_session, - study_id=study_id, - profile_ids=agent_profile_uuids, - baseline_profile_id=baseline_profile_uuid, - ) - db_session.commit() return JsonResponseWithStatus( @@ -392,21 +372,9 @@ def get_study_details( "engagement_rate": float(assignment.engagement_rate) if assignment.engagement_rate else 0.0 }) - # Agent-profile arms attached to this study. Empty for completion-only - # studies, which is the backwards-compatible default. - agent_profiles = [ - { - "profile_id": str(link.profile_id), - "name": link.profile.name if link.profile else None, - "model": link.profile.model if link.profile else None, - "framework_version": ( - link.profile.framework_version if link.profile else None - ), - "is_active": link.profile.is_active if link.profile else None, - "is_baseline": link.is_baseline, - } - for link in crud.list_study_agent_profiles(db_session, study_uuid) - ] + # Agent-profile arms were removed by the wave-1 consolidation; a + # completion study has none. + agent_profiles: list[dict] = [] return JsonResponseWithStatus( status_code=200, @@ -606,15 +574,23 @@ def evaluate_study( if not study: raise HTTPException(status_code=404, detail="Study not found") - # Get study results by configuration + # Get study results by configuration. + # + # ``acceptance_rate`` here is the *legacy completion* acceptance + # (``had_generation.was_accepted``), a real completion-product producer — + # it is NOT the research edit acceptance (which has no producer and is + # rendered unavailable on the research surfaces). Unresolved generations + # (NULL ``was_accepted``) are excluded from the manual ratio rather than + # counted as rejections (L07). evaluation_query = """ SELECT cah.assigned_config_id, COUNT(DISTINCT cah.user_id) as total_users, COUNT(DISTINCT mq.user_id) as active_users, COUNT(mq.meta_query_id) as total_queries, - AVG(CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END) as acceptance_rate, - COUNT(CASE WHEN hg.was_accepted THEN 1 END) as total_accepted, + AVG(CASE WHEN hg.was_accepted THEN 1.0 END) + FILTER (WHERE hg.was_accepted IS NOT NULL) as acceptance_rate, + COUNT(CASE WHEN hg.was_accepted IS TRUE THEN 1 END) as total_accepted, COUNT(hg.meta_query_id) as total_generations, AVG(hg.generation_time) as avg_generation_time, AVG(hg.confidence) as avg_confidence, @@ -652,7 +628,11 @@ def evaluate_study( "activation_rate": (row.active_users or 0) / max(row.total_users, 1), "total_queries": row.total_queries or 0, "total_generations": row.total_generations or 0, - "acceptance_rate": float(row.acceptance_rate) if row.acceptance_rate else 0.0, + "acceptance_rate": ( + float(row.acceptance_rate) + if row.acceptance_rate is not None + else None + ), "total_accepted": row.total_accepted or 0, "avg_generation_time": float(row.avg_generation_time) if row.avg_generation_time else 0.0, "avg_confidence": float(row.avg_confidence) if row.avg_confidence else 0.0, @@ -676,14 +656,23 @@ def evaluate_study( config_acceptance = config_data["metrics"]["acceptance_rate"] config_generation_time = config_data["metrics"]["avg_generation_time"] - # Calculate uplift percentages - acceptance_uplift = ((config_acceptance - baseline_acceptance) / max(baseline_acceptance, 0.001)) * 100 + # Calculate uplift percentages (None when the acceptance + # rate is unavailable because no generation was decided). + acceptance_uplift = ( + ((config_acceptance - baseline_acceptance) / max(baseline_acceptance, 0.001)) * 100 + if config_acceptance is not None and baseline_acceptance is not None + else None + ) generation_time_change = ((config_generation_time - baseline_generation_time) / max(baseline_generation_time, 1)) * 100 config_data["vs_baseline"] = { "acceptance_rate_uplift_pct": acceptance_uplift, "generation_time_change_pct": generation_time_change, - "is_better_acceptance": config_acceptance > baseline_acceptance, + "is_better_acceptance": ( + config_acceptance > baseline_acceptance + if config_acceptance is not None and baseline_acceptance is not None + else None + ), "is_faster": config_generation_time < baseline_generation_time } @@ -718,18 +707,19 @@ def evaluate_study_agents( """ Evaluate an A/B study's *agent* arms. - The agent-side mirror of ``evaluate_study``: aggregates agent telemetry - (``agent_task`` + ``agent_event``) per snapshotted profile and reports - uplift against the baseline arm. + The agent-side mirror of ``evaluate_study``: aggregates canonical agent + telemetry per snapshotted profile and reports uplift against the baseline + arm. A task counts only when its immutable ``study_id`` and ``profile_id`` snapshots identify this study. Legacy tasks without those snapshots are intentionally excluded rather than inferred from a mutable profile name or - a time window. + a time window. Canonical events join their task by the explicit phase-05 + binding (``research_event.agent_run_id = agent_task.external_run_id``), never + by an ACP session id or a time window. - Because both telemetry paths write into the same ``agent_event`` table, the - same aggregation works whether an arm ran Goose, Codex, or the built-in - ``code4me2-agent`` runtime — which is the whole point of the unified schema. + Edit acceptance has no canonical producer, so it is reported unavailable + (see ``edit_acceptance_rate: None``) rather than inferred. Admin only endpoint. """ @@ -781,20 +771,29 @@ def evaluate_study_agents( """ task_rows = db_session.execute(text(task_query), window).fetchall() - # Per-arm event-level aggregates, kept as a separate query so the task - # aggregates above aren't inflated by the event-row fan-out. + # Canonical per-arm event aggregates. Events join their task by the + # explicit phase-05 binding (research_event.agent_run_id = + # agent_task.external_run_id), never by an ACP session id or time window. event_query = """ SELECT t.profile_id, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_request') AS model_requests, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'model_call') AS model_calls, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_request') AS tool_requests, - COUNT(e.event_id) FILTER (WHERE e.event_type = 'tool_call') AS tool_calls, - COUNT(e.event_id) FILTER (WHERE e.event_type IN ('tool_failed', 'error')) AS failures, - AVG(e.latency_ms) FILTER (WHERE e.event_type = 'model_call') AS avg_model_latency_ms, - SUM(e.total_tokens) AS total_tokens + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS model_calls, + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'tool_call' + ) AS tool_calls, + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'tool_failed' + ) AS failures, + AVG((e.envelope_json -> 'metrics' ->> 'latency_ms')::float) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS avg_model_latency_ms, + SUM((e.envelope_json -> 'metrics' ->> 'usage_tokens')::bigint) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS total_tokens FROM agent_task t - LEFT JOIN agent_event e ON e.task_id = t.task_id + LEFT JOIN research_event e ON e.agent_run_id = t.external_run_id WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL GROUP BY t.profile_id """ @@ -803,26 +802,10 @@ def evaluate_study_agents( for row in db_session.execute(text(event_query), window).fetchall() } - # Human-in-the-loop signal: what the developer actually did with the - # agent's proposals. This is the metric group-5 couldn't report at all, - # and it's arguably the most important one — an arm that proposes more - # edits but gets fewer accepted is worse, not better. - edit_query = """ - SELECT - t.profile_id, - COUNT(ed.edit_id) AS total_edits, - COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS TRUE) AS accepted_edits, - COUNT(ed.edit_id) FILTER (WHERE ed.was_accepted IS FALSE) AS rejected_edits, - COUNT(ed.edit_id) FILTER (WHERE ed.was_modified IS TRUE) AS modified_edits - FROM agent_task t - LEFT JOIN agent_edit ed ON ed.task_id = t.task_id - WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL - GROUP BY t.profile_id - """ - edit_rows = { - str(row.profile_id): row - for row in db_session.execute(text(edit_query), window).fetchall() - } + # Edit acceptance has no canonical producer (the contract records it only + # "when truly exposed"), so it is reported unavailable rather than + # computed from anything else. There is no second, legacy edit source. + edit_rows: dict[str, object] = {} def _f(value) -> float: return float(value) if value is not None else 0.0 @@ -835,10 +818,7 @@ def _i(value) -> int: for row in task_rows: events = event_rows.get(str(row.profile_id)) - edits = edit_rows.get(str(row.profile_id)) total_tasks = _i(row.total_tasks) - total_edits = _i(edits.total_edits) if edits else 0 - accepted_edits = _i(edits.accepted_edits) if edits else 0 arm = { "profile_id": str(row.profile_id), @@ -855,28 +835,20 @@ def _i(value) -> int: "avg_steps": _f(row.avg_steps), "total_input_tokens": _i(row.total_input_tokens), "total_output_tokens": _i(row.total_output_tokens), - "model_requests": _i(events.model_requests) if events else 0, "model_calls": _i(events.model_calls) if events else 0, - "tool_requests": _i(events.tool_requests) if events else 0, "tool_calls": _i(events.tool_calls) if events else 0, "failures": _i(events.failures) if events else 0, "avg_model_latency_ms": ( _f(events.avg_model_latency_ms) if events else 0.0 ), "total_tokens": _i(events.total_tokens) if events else 0, - "total_edits": total_edits, - "accepted_edits": accepted_edits, - "rejected_edits": _i(edits.rejected_edits) if edits else 0, - "modified_edits": _i(edits.modified_edits) if edits else 0, - # Undecided edits are excluded from the denominator: an edit - # the developer hasn't ruled on yet is not a rejection. - "edit_acceptance_rate": ( - accepted_edits - / max( - accepted_edits + (_i(edits.rejected_edits) if edits else 0), - 1, - ) - ), + # No canonical edit producer exists, so edit acceptance is + # explicitly unavailable (never computed from tool success). + "total_edits": None, + "accepted_edits": None, + "rejected_edits": None, + "modified_edits": None, + "edit_acceptance_rate": None, }, } @@ -895,9 +867,9 @@ def _i(value) -> int: "completion_rate_change_pct": _pct_change( metrics["completion_rate"], base["completion_rate"] ), - "edit_acceptance_change_pct": _pct_change( - metrics["edit_acceptance_rate"], base["edit_acceptance_rate"] - ), + # Edit acceptance is unavailable (no canonical producer), so + # its uplift is reported as unknown rather than zero. + "edit_acceptance_change_pct": None, "avg_steps_change_pct": _pct_change( metrics["avg_steps"], base["avg_steps"] ), @@ -907,9 +879,7 @@ def _i(value) -> int: "is_better_completion": ( metrics["completion_rate"] > base["completion_rate"] ), - "is_better_acceptance": ( - metrics["edit_acceptance_rate"] > base["edit_acceptance_rate"] - ), + "is_better_acceptance": None, "is_faster": ( metrics["avg_model_latency_ms"] < base["avg_model_latency_ms"] ), diff --git a/src/backend/routers/research/__init__.py b/src/backend/routers/research/__init__.py new file mode 100644 index 0000000..9df80a1 --- /dev/null +++ b/src/backend/routers/research/__init__.py @@ -0,0 +1,91 @@ +"""Research-platform sub-router (Issue 01+). + +Mounted at ``/api/research``. The study protocol publication API is +researcher/admin-scoped: it defines experimental intent, so it is never exposed +to participants. Capability qualification is confined to the admin-only +``/packages`` operator/build surface; there is no public self-attestation route. +""" + +from fastapi import APIRouter + +from .agents import router as agents_router +from .bootstrap import router as bootstrap_router +from .join import router as join_router +from .operations import operations_router as pilot_operations_router +from .packages import packages_router +from .participants import router as participants_router +from .providers import router as providers_router +from .read_models import operations_router as read_operations_router +from .researchers import router as researchers_router +from .sessions import router as sessions_router +from .studies import router as studies_router +from .telemetry import router as telemetry_router + +router = APIRouter() +router.include_router( + studies_router, + prefix="/studies", + tags=["Research Studies"], +) +# Participant self-enrollment by shared join code. ``/join/{code}`` is readable +# by any authenticated user; ``POST /join`` enrolls the caller. +router.include_router( + join_router, + prefix="/join", + tags=["Research Onboarding"], +) +# Administrator enablement of researcher accounts (the single researcher +# authority; no per-study role grants). +router.include_router( + researchers_router, + prefix="/researchers", + tags=["Research Researchers"], +) +# Administrator-managed provider connections plus a researcher's granted subset. +router.include_router( + providers_router, + tags=["Research Provider Connections"], +) +router.include_router( + agents_router, + prefix="/agents", + tags=["Research Agents"], +) +router.include_router( + participants_router, + prefix="/participants", + tags=["Research Participants"], +) +router.include_router( + bootstrap_router, + prefix="/bootstrap", + tags=["Research Bootstrap"], +) +router.include_router( + sessions_router, + prefix="/sessions", + tags=["Research Sessions"], +) +router.include_router( + telemetry_router, + prefix="/telemetry", + tags=["Research Telemetry"], +) +# One operations/read API surface: the RBAC-scoped researcher read models +# (revision summaries, enrollment/exposure coverage, telemetry coverage and +# metrics) and the admin-only pilot/release/kill-switch operations are merged +# into a single router mounted exactly once under /operations. Their handler +# paths are unchanged; authorization remains per-route. +operations_router = APIRouter() +operations_router.include_router(read_operations_router) +operations_router.include_router(pilot_operations_router) +router.include_router( + operations_router, + prefix="/operations", + tags=["Research Operations"], +) +router.include_router( + packages_router, + prefix="/packages", + tags=["Research Packages"], +) diff --git a/src/backend/routers/research/access.py b/src/backend/routers/research/access.py new file mode 100644 index 0000000..1833071 --- /dev/null +++ b/src/backend/routers/research/access.py @@ -0,0 +1,276 @@ +"""Researcher/administrator authorization helpers (consolidation phase 01). + +There is exactly one researcher authority: the ``user.can_research`` flag that an +administrator enables (P3). There is no per-study role table. Study ownership is +``study.created_by == user.user_id``; profile ownership is +``agent_profile.owner_user_id``. Administrators bypass every ownership check. +No route may let a non-administrator change ``can_research`` or ``is_admin``. + +This is also the single funded-access gate (phase 03): every funded operation +(participant relay inference, managed ACP inference, self-report run +creation/replay, bootstrap session creation, ACP grant minting) resolves the same +current server state here — the account is a participant, has a live (``ACTIVE``) +enrollment, the research study window is open, and no operator kill switch is +engaged. Server time is authoritative; a previously issued capability never +substitutes for this live re-check. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Callable, Optional + +from fastapi import HTTPException + +from backend.routers.analytics.auth_utils import AuthenticatedUser +from database.db_schemas import Study as StudyRow +from database.research_schemas import ResearchSessionV1 as ResearchSessionRow +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.study.protocol import store as protocol_store + +__all__ = [ + "is_researcher", + "require_researcher", + "is_owner", + "require_owner", + "is_study_owner", + "require_study_owner", + "FundedAccessRefused", + "require_live_enrollment", + "ResearchBinding", + "resolve_research_binding", +] + + +def is_researcher(user: AuthenticatedUser) -> bool: + """Whether the caller may act as a researcher (admin or enabled account).""" + return bool( + getattr(user, "is_admin", False) or getattr(user, "can_research", False) + ) + + +def require_researcher(user: AuthenticatedUser) -> AuthenticatedUser: + """Reject a caller who is neither an administrator nor an enabled researcher.""" + if not is_researcher(user): + raise HTTPException( + status_code=403, + detail={ + "code": "RESEARCHER_REQUIRED", + "message": ( + "an administrator must enable this account for research" + ), + }, + ) + return user + + +def is_owner(user: AuthenticatedUser, owner_user_id) -> bool: + """Whether ``user`` owns a row (administrators own everything).""" + if getattr(user, "is_admin", False): + return True + if owner_user_id is None: + return False + return str(owner_user_id) == str(user.user_id) + + +def require_owner(user: AuthenticatedUser, owner_user_id, *, subject: str = "resource"): + """Reject a non-owner, non-administrator caller.""" + if not is_owner(user, owner_user_id): + raise HTTPException( + status_code=403, + detail={ + "code": "FORBIDDEN_OWNER", + "message": f"not authorized for this {subject}", + }, + ) + + +def is_study_owner(user: AuthenticatedUser, study_row) -> bool: + """Study ownership is ``study.created_by`` (admin bypasses).""" + return is_owner(user, getattr(study_row, "created_by", None)) + + +def require_study_owner(user: AuthenticatedUser, study_row): + """Reject a caller who neither owns the study nor is an administrator.""" + if not is_study_owner(user, study_row): + raise HTTPException( + status_code=403, + detail={ + "code": "FORBIDDEN_STUDY", + "message": "not authorized for this study", + }, + ) + + +# --------------------------------------------------------------------------- +# Funded-access gate (phase 03; folded from research/runtime/funding.py) +# --------------------------------------------------------------------------- + + +class FundedAccessRefused(Exception): + """A typed refusal for a funded path (mapped to HTTP 403 by routers).""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def require_live_enrollment( + db: Any, + *, + account_id: Optional[uuid.UUID], + study_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, +): + """Return the account's live enrollment or raise :class:`FundedAccessRefused`. + + ``study_id`` (when given) must match the live enrollment's study, so a task + minted for one study cannot be funded by an enrollment in another. + """ + if account_id is None: + raise FundedAccessRefused( + "ACCOUNT_REQUIRED", "funded research use requires an authenticated account" + ) + + participant_row = identity_store.get_participant_by_account(db, account_id) + if participant_row is None: + raise FundedAccessRefused( + "NOT_A_PARTICIPANT", "this account has no research participant mapping" + ) + + active = None + for row in identity_store.list_enrollments(db, participant_row.participant_id): + if row.status == EnrollmentStatus.ACTIVE.value: + active = row + break + if active is None: + raise FundedAccessRefused( + "ENROLLMENT_NOT_ACTIVE", + "this account has no active study enrollment", + ) + + if study_id is not None and active.study_id != study_id: + raise FundedAccessRefused( + "ENROLLMENT_STUDY_MISMATCH", + "the active enrollment is not for this study", + ) + + study = db.get(StudyRow, active.study_id) + if not protocol_store.research_study_is_open(study, _now(now)): + # Lazy terminal sweep: a study that ended or was terminated is marked + # completed here (COMPLETED enrollments, revoked sessions, bumped epoch) + # so a previously issued capability cannot keep funded access alive. + _sweep_ended_study(db, study, active.study_id, now=_now(now)) + raise FundedAccessRefused( + "STUDY_NOT_OPEN", + "the study is not currently open (not started, ended, or terminated)", + ) + + if kill_switch_check is not None and kill_switch_check(): + raise FundedAccessRefused( + "KILL_SWITCH_ENGAGED", + "an operator kill switch is engaged for this study", + ) + + return active + + +def _sweep_ended_study(db, study, study_id, *, now: datetime) -> None: + """Complete enrollments for a study that is no longer open for execution. + + Only sweeps when the study is an ended/terminated research study (not merely + pre-start), so a scheduled future study's pre-start enrollments are left + intact. + """ + if study is None or not getattr(study, "is_research", False): + return + ends_at = getattr(study, "ends_at", None) + ended = ends_at is not None and now > ends_at + terminated = not getattr(study, "is_active", False) + if not (ended or terminated): + return + try: + identity_store.complete_enrollments_for_study(db, study_id, now=now) + except Exception as error: # noqa: BLE001 - the refusal above is decisive + logging.warning( + "[Funding] could not complete enrollments for closed study %s: %s", + study_id, + error, + ) + + +# --------------------------------------------------------------------------- +# Explicit research attribution (phase 05) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ResearchBinding: + """An agent task's explicit research attribution. + + Resolved server-side from the *authorized* account and the frozen + assignment's study; never supplied by a caller and never inferred from a + time window. ``research_session_id`` is ``None`` when the enrollment has no + single active session (none, or more than one open window) — an explicit + "unknown context" state rather than a guessed one. + """ + + enrollment_id: uuid.UUID + study_revision_id: uuid.UUID + research_session_id: Optional[uuid.UUID] + + +def resolve_research_binding( + db: Any, *, account_id: Optional[uuid.UUID], study_id: Optional[uuid.UUID] +) -> Optional[ResearchBinding]: + """Resolve the account's live research attribution for ``study_id``. + + Returns ``None`` (no research context) when the account has no participant + mapping or no ACTIVE enrollment in that study. A single active session for + the enrollment is bound explicitly; zero or multiple active sessions leave + ``research_session_id`` unknown. + """ + if account_id is None or study_id is None: + return None + participant_row = identity_store.get_participant_by_account(db, account_id) + if participant_row is None: + return None + active = next( + ( + row + for row in identity_store.list_enrollments(db, participant_row.participant_id) + if row.status == EnrollmentStatus.ACTIVE.value + and getattr(row, "study_id", None) == study_id + ), + None, + ) + if active is None: + return None + + session_rows = ( + db.query(ResearchSessionRow) + .filter( + ResearchSessionRow.enrollment_id == active.enrollment_id, + ResearchSessionRow.state.notin_(("ended", "revoked")), + ) + .all() + ) + research_session_id = ( + session_rows[0].session_id if len(session_rows) == 1 else None + ) + return ResearchBinding( + enrollment_id=active.enrollment_id, + study_revision_id=active.study_revision_id, + research_session_id=research_session_id, + ) + diff --git a/src/backend/routers/research/agents.py b/src/backend/routers/research/agents.py new file mode 100644 index 0000000..aed6082 --- /dev/null +++ b/src/backend/routers/research/agents.py @@ -0,0 +1,538 @@ +"""Admin API for the agent registry and capability contract. + +Mounted under ``/api/research/agents``. Every endpoint is admin-only. The +handlers are intentionally thin: registration, transitions, platform selection +and coverage live in :mod:`research.study.agents.registry`, and the SQLAlchemy session +is supplied by ``App.get_db_session`` and passed to the store helpers. + +Release records are digest-pinned and immutable. Editing a release's bytes +requires a new ``release_id``; retirement preserves historical references. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.exc import IntegrityError + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from database import crud +from research.study.agents import store +from research.study.agents.distributions import ( + distribution_supported_platforms, + resolve_distribution_view, +) +from research.study.agents.enums import DistributionMode, RegistryReasonCode +from research.study.agents.manifest_import import ( + ManifestImportError, + build_manifest_release, +) +from research.study.agents.models import ( + AgentReleaseV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + CapabilitySnapshotV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.study.agents.registry import ( + AgentRegistry, + coverage_report, + verified_approval_options, +) + +router = APIRouter() + + +class ReleaseRegisterRequest(BaseModel): + """Register a digest-pinned agent release. + + ``AgentReleaseV1`` owns artifact/adapter identity only. Qualification is + derived from verified conformance evidence, so a payload carrying a + ``qualification_status`` is rejected rather than trusted. + """ + + release: AgentReleaseV1 + + +class ResolveArtifactRequest(BaseModel): + """Select the distribution artifact for one participant platform.""" + + os: str + arch: str + + +class SnapshotUploadRequest(BaseModel): + """Upload a declared-vs-observed capability snapshot for a release.""" + + snapshot: CapabilitySnapshotV1 + + +class ReleaseImportRequest(BaseModel): + """Import one build runtime manifest as a digest-pinned release. + + ``manifest`` is the packaging pipeline's manifest document. ``artifact_root`` + is an optional directory the manifest's relative ``archive`` paths resolve + against; when a listed archive is present there its real digest/size are + computed and the declared digest is verified. ``artifact_sizes`` supplies + sizes for archives that are not available on disk -- a size is never guessed. + """ + + manifest: dict[str, Any] + artifact_root: Optional[str] = None + artifact_sizes: dict[str, int] = Field(default_factory=dict) + + +def _registry_from_db(db: Any) -> AgentRegistry: + """Load a registry view so domain checks see existing releases.""" + registry = AgentRegistry() + for row in store.list_releases(db): + registry.register_release(store.row_to_release(row)) + return registry + + +def _registry_with_release(release: AgentReleaseV1) -> AgentRegistry: + registry = AgentRegistry() + registry.register_release(release) + return registry + + +def _issue_payload(issue: Any) -> dict[str, Any]: + return issue.model_dump(mode="json") if issue is not None else {} + + +def _release_for_profile(db: Any, profile: Any): + """Rehydrate a PACKAGED distribution's release, or ``None``.""" + release_id = getattr(profile, "release_id", None) + if not release_id: + return None + row = store.get_release(db, release_id) + if row is None: + return None + return store.row_to_release(row) + + +def _distribution_payload(db: Any, profile: Any) -> dict[str, Any]: + """The derived, read-only distribution view the UI consumes. + + Exposes the resolved ``release_id`` and ``version``, the DERIVED ``verified`` + boolean and the release's supported platforms. No secret is included: the + provider block carries only non-secret identity (the connection id), never + the endpoint URL or the secret reference/value. + """ + release = _release_for_profile(db, profile) + view = resolve_distribution_view( + profile, release, distribution_id=profile.profile_id + ) + release_row = ( + store.get_release(db, profile.release_id) + if getattr(profile, "release_id", None) + else None + ) + return { + "distribution_id": str(profile.profile_id), + "name": profile.name, + "distribution_mode": view.distribution_mode, + "release_id": view.release_id, + "release_version": view.version, + "verified": view.verified, + "supported_platforms": distribution_supported_platforms(release), + # Approval options the release's conformance evidence actually verifies; + # the editor must not offer an option this list does not contain. + "verified_approval_options": verified_approval_options( + getattr(release_row, "release_json", None) + if release_row is not None + else None + ), + "agent_package": view.agent_package, + "agent_command": view.agent_command, + "agent_command_args": view.agent_command_args, + "provider": { + "framework_version": profile.framework_version, + "model": profile.model, + "connection_id": ( + str(profile.connection_id) + if getattr(profile, "connection_id", None) is not None + else None + ), + }, + } + + +@router.post("/releases", summary="Register a digest-pinned agent release") +def register_release( + payload: ReleaseRegisterRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Register a release; exact duplicate digest identity is rejected.""" + require_admin(current_user) + + if "qualification_status" in payload.release.model_fields_set: + raise HTTPException( + status_code=422, + detail=( + "qualification_status is derived from verified conformance " + "evidence and cannot be supplied by a caller" + ), + ) + + db = app.get_db_session() + try: + registry = _registry_from_db(db) + result = registry.register_release(payload.release) + if not result.accepted: + status_code = ( + 409 + if result.issue + and result.issue.code == RegistryReasonCode.DUPLICATE_RELEASE + else 422 + ) + raise HTTPException( + status_code=status_code, detail=_issue_payload(result.issue) + ) + try: + row = store.upsert_release(db, payload.release) + except IntegrityError as error: + db.rollback() + raise HTTPException( + status_code=409, + detail="release manifest digest already exists", + ) from error + return JsonResponseWithStatus( + status_code=201, + content={ + "accepted": True, + "release": store.release_summary(row), + }, + ) + finally: + db.close() + + +@router.post( + "/releases/import", + summary="Register release(s) from a build runtime manifest", +) +def import_release( + payload: ReleaseImportRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Import a build manifest as an idempotent, digest-pinned release. + + The release identity is derived from the manifest bytes, so re-importing the + same manifest returns the existing release (never a duplicate/409). A + manifest whose declared digest disagrees with an archive available under + ``artifact_root`` is rejected loudly. Admin-only; takes a JSON body only, so + CI can call it without any interactive prompt. + """ + require_admin(current_user) + + try: + plan = build_manifest_release( + payload.manifest, + artifact_root=payload.artifact_root, + artifact_sizes=payload.artifact_sizes, + ) + except ManifestImportError as error: + raise HTTPException( + status_code=422, + detail={ + "code": error.code, + "message": error.message, + "field": error.field, + }, + ) from error + + skipped = [ + {"platform": item.platform, "archive": item.archive, "reason": item.reason} + for item in plan.skipped + ] + + db = app.get_db_session() + try: + registry = _registry_from_db(db) + existing = registry.get_release( + plan.release.agent_id, plan.release.release_id + ) + if existing is not None: + if existing.digest_identity != plan.release.digest_identity: + raise HTTPException( + status_code=409, + detail={ + "code": RegistryReasonCode.DUPLICATE_RELEASE.value, + "message": ( + "release_id is already registered for this agent with " + "a different manifest digest" + ), + "field": "release_id", + }, + ) + row = store.get_release(db, plan.release.release_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "accepted": True, + "created": False, + "release": store.release_summary(row) if row else {}, + "skipped_artifacts": skipped, + }, + ) + + result = registry.register_release(plan.release) + if not result.accepted: + raise HTTPException( + status_code=422, detail=_issue_payload(result.issue) + ) + try: + row = store.upsert_release(db, plan.release) + except IntegrityError as error: + db.rollback() + raise HTTPException( + status_code=409, + detail="release manifest digest already exists", + ) from error + return JsonResponseWithStatus( + status_code=201, + content={ + "accepted": True, + "created": True, + "release": store.release_summary(row), + "skipped_artifacts": skipped, + }, + ) + finally: + db.close() + + +@router.get("/distributions", summary="List distributions and their derived release view") +def list_distributions( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """List every ``AgentProfile`` as a distribution with its resolved release. + + Readable by any authenticated user (it exposes no secret) so a researcher can + see which distributions are VERIFIED before authoring a condition. + """ + db = app.get_db_session() + try: + profiles = crud.list_agent_profiles(db) + return JsonResponseWithStatus( + status_code=200, + content={ + "distributions": [ + _distribution_payload(db, profile) for profile in profiles + ] + }, + ) + finally: + db.close() + + +@router.get( + "/distributions/{distribution_id}", + summary="Fetch one distribution's derived release view", +) +def get_distribution( + distribution_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + profile = crud.get_agent_profile_by_id(db, distribution_id) + if profile is None: + raise HTTPException(status_code=404, detail="Distribution not found") + return JsonResponseWithStatus( + status_code=200, content={"distribution": _distribution_payload(db, profile)} + ) + finally: + db.close() + + +@router.get("/releases", summary="List registered agent releases") +def list_releases( + agent_id: Optional[str] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + rows = store.list_releases(db, agent_id) + return JsonResponseWithStatus( + status_code=200, + content={"releases": [store.release_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get("/releases/{release_id}", summary="Fetch one registered release") +def get_release( + release_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + row = store.get_release(db, release_id) + if row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "release": store.release_summary(row), + "model": store.row_to_release(row).model_dump(mode="json"), + }, + ) + finally: + db.close() + + +@router.post( + "/releases/{release_id}/resolve", + summary="Resolve the distribution artifact for a platform", +) +def resolve_artifact( + release_id: str, + payload: ResolveArtifactRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return the exact platform artifact or a typed compatibility block.""" + require_admin(current_user) + + db = app.get_db_session() + try: + row = store.get_release(db, release_id) + if row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + + release = store.row_to_release(row) + registry = _registry_with_release(release) + result = registry.resolve_distribution(release, payload.os, payload.arch) + if not result.resolved: + raise HTTPException( + status_code=409, detail=_issue_payload(result.issue) + ) + content: dict[str, Any] = { + "resolved": True, + "distribution_mode": result.distribution_mode.value, + "artifact": ( + result.artifact.model_dump(mode="json") + if result.artifact is not None + else None + ), + } + if result.distribution_mode == DistributionMode.BYOA_EXTERNAL: + # A BYOA release resolves to a participant-installable identity, not + # a digest-pinned artifact. + content["agent_command"] = result.agent_command + content["agent_command_args"] = result.agent_command_args + content["agent_package"] = result.agent_package + return JsonResponseWithStatus(status_code=200, content=content) + finally: + db.close() + + +@router.post( + "/releases/{release_id}/snapshots", + summary="Upload an immutable capability snapshot", +) +def upload_snapshot( + release_id: str, + payload: SnapshotUploadRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Store a declared-vs-observed snapshot linked to the exact release.""" + require_admin(current_user) + + if payload.snapshot.release_id != release_id: + raise HTTPException( + status_code=422, + detail="snapshot.release_id does not match the path release_id", + ) + + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + row = store.insert_snapshot(db, payload.snapshot) + return JsonResponseWithStatus( + status_code=201, + content={"snapshot": store.snapshot_summary(row)}, + ) + finally: + db.close() + + +@router.get( + "/releases/{release_id}/snapshots", + summary="List capability snapshots for a release", +) +def list_snapshots( + release_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + rows = store.list_snapshots(db, release_id) + return JsonResponseWithStatus( + status_code=200, + content={"snapshots": [store.snapshot_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get( + "/releases/{release_id}/coverage", + summary="Summarize declared vs observed capability coverage", +) +def coverage( + release_id: str, + snapshot_id: Optional[uuid.UUID] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return the coverage matrix for one snapshot (default: the latest).""" + require_admin(current_user) + + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + + rows = list(store.list_snapshots(db, release_id)) + if snapshot_id is not None: + rows = [ + row + for row in rows + if store.row_to_snapshot(row).snapshot_id == snapshot_id + ] + if not rows: + raise HTTPException( + status_code=404, detail="No capability snapshot found for release" + ) + + snapshot = store.row_to_snapshot(rows[-1]) + return JsonResponseWithStatus( + status_code=200, + content=coverage_report(snapshot).model_dump(mode="json"), + ) + finally: + db.close() diff --git a/src/backend/routers/research/bootstrap.py b/src/backend/routers/research/bootstrap.py new file mode 100644 index 0000000..0a9b06b --- /dev/null +++ b/src/backend/routers/research/bootstrap.py @@ -0,0 +1,582 @@ +"""Participant bootstrap API: manifest and exposure receipts. + +Mounted under ``/api/research/bootstrap``. Handlers are intentionally thin: the +assignment, exposure, and bootstrap composition logic lives in +:mod:`research.runtime.assignment` and :mod:`research.runtime.bootstrap`. Participants +authenticate with the normal ``get_current_user`` flow and only ever obtain a +signed, secret-free manifest for their own active enrollment. +""" + +from __future__ import annotations + +import os +import random +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Callable, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, +) +from database import crud +from research.analysis.operations import store as operations_store +from research.compatibility.enums import CapabilityId, CapabilityState +from research.compatibility.evaluate import evaluate_compatibility +from research.compatibility.models import ( + AcpCapabilityReceiptV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + AgentIdentity, + CompatibilityRequest, + EnvironmentTuple, + RequiredCapability, +) +from research.participants import identity as identity_store +from research.runtime.assignment import store as assignment_store +from research.runtime.assignment.enums import AllocationOutcome, ExposureOutcome +from research.runtime.assignment.exposure import record_exposure +from research.runtime.assignment.models import ( + ExposureEnvironment, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.runtime.assignment.service import allocate +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.models import ( + BootstrapAgentProfile, + ResearchSessionRef, + SessionCapability, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.runtime.bootstrap.service import ( + BootstrapSigningContext, + compose_bootstrap, +) +from research.runtime.sessions import store as session_store +from research.runtime.sessions.enums import SessionState +from research.runtime.sessions.models import ResearchSessionV1 +from research.study.agents import store as registry_store +from research.study.protocol import store as protocol_store +from research.study.protocol.models import StudyProtocolV1 + +router = APIRouter() + +# The signing secret is mandatory: there is no development fallback. If it is +# unset the router refuses to issue or verify any capability/manifest rather +# than signing with a predictable, hardcoded secret. +BOOTSTRAP_SIGNING_SECRET: Optional[str] = os.environ.get("BOOTSTRAP_SIGNING_SECRET") +_SIGNER: Optional[BootstrapSigningContext] = ( + BootstrapSigningContext(secret=BOOTSTRAP_SIGNING_SECRET) + if BOOTSTRAP_SIGNING_SECRET + else None +) + +AUDIENCE = "research-runtime" + + +def _require_signer() -> BootstrapSigningContext: + """Return the configured signer or raise a typed refusal.""" + if _SIGNER is None: + raise HTTPException( + status_code=503, + detail={ + "code": "SIGNING_SECRET_MISSING", + "message": ( + "BOOTSTRAP_SIGNING_SECRET is not configured; refusing to " + "issue a bootstrap manifest" + ), + }, + ) + return _SIGNER + + +def _kill_switch_for_scope( + db: Any, + *, + study_id: Optional[uuid.UUID], + revision_id: Optional[uuid.UUID], + enrollment_id: Optional[uuid.UUID], +) -> Callable[[], bool]: + """DB-backed kill-switch predicate for one bootstrap/exposure scope.""" + return operations_store.db_kill_switch_check( + db, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + ) + + +def _verify_exposure_capability( + capability: SessionCapability, + *, + enrollment: Any, + assignment: Any, + now: datetime, +) -> None: + """Verify the exposure capability is bound to this enrollment/assignment.""" + if not BOOTSTRAP_SIGNING_SECRET: + raise HTTPException( + status_code=503, + detail={ + "code": "SIGNING_SECRET_MISSING", + "message": ( + "BOOTSTRAP_SIGNING_SECRET is not configured; refusing to " + "verify a session capability" + ), + }, + ) + verification = verify_capability( + capability, + BOOTSTRAP_SIGNING_SECRET, + expected_audience=AUDIENCE, + expected_scope=[], + now=now, + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=enrollment.enrollment_id, + expected_revision_id=assignment.study_revision_id, + ) + if not verification.ok: + raise HTTPException( + status_code=403, + detail={ + "code": "CAPABILITY_INVALID", + "message": verification.message or "session capability is invalid", + "capability_reason": verification.reason.value, + }, + ) + + +class EnvironmentReport(BaseModel): + """The participant's installed host tuple.""" + + os: str + arch: str + ide_build: Optional[str] = None + plugin_version: Optional[str] = None + host_kind: Optional[str] = None + + +class ResearchSessionRequest(BaseModel): + """Request a bootstrap manifest for an active enrollment.""" + + enrollment_id: uuid.UUID + # Opaque execution-context id for this project/window. Required so two + # simultaneous windows get distinct sessions; identical values are + # idempotent. Never a filesystem path. + context_id: str = Field(min_length=1, max_length=200) + environment: EnvironmentReport + capability_receipt: Optional[AcpCapabilityReceiptV1] = None + compatibility_receipt_id: Optional[uuid.UUID] = None + + +class ExposureRequest(BaseModel): + """Report an exposure (or a launch failure) for an assignment.""" + + # The exposure is authorized by the same session capability issued at + # bootstrap; the capability's enrollment must match the assignment's. + capability: SessionCapability + enrollment_id: uuid.UUID + assignment_id: uuid.UUID + environment: ExposureEnvironment + outcome: ExposureOutcome + idempotency_key: str = Field(min_length=1) + agent_release_id: Optional[str] = None + artifact_digest: Optional[str] = None + adapter_version: Optional[str] = None + observed_configuration: dict[str, Any] = Field(default_factory=dict) + evidence_digest: Optional[str] = None + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue_payload(issue: Any) -> dict[str, Any]: + return issue.model_dump(mode="json") if issue is not None else {} + + +def _owned_enrollment(db: Any, current_user: AuthenticatedUser, enrollment_id: uuid.UUID): + participant_row = identity_store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + row = identity_store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + enrollment = identity_store.row_to_enrollment(row) + if enrollment.participant_id != participant_row.participant_id: + raise HTTPException(status_code=404, detail="Enrollment not found") + return enrollment + + +def _evaluate_compatibility( + payload: ResearchSessionRequest, + protocol: StudyProtocolV1, + release: Any, + db: Any, +) -> tuple[Any, Optional[str]]: + """Evaluate compatibility from an inline receipt or a stored receipt id.""" + receipt = payload.capability_receipt + receipt_ref: Optional[str] = None + if receipt is not None: + receipt_ref = str(receipt.receipt_id) + elif payload.compatibility_receipt_id is not None: + row = _compatibility_store().get_receipt(db, payload.compatibility_receipt_id) + if row is not None: + receipt = _compatibility_store().row_to_receipt(row) + receipt_ref = str(payload.compatibility_receipt_id) + + if receipt is None: + return None, receipt_ref + + requirements: list[RequiredCapability] = [] + for expectation in protocol.environment_requirements.required_capabilities: + try: + capability = CapabilityId(str(expectation.capability).strip().upper()) + require_state = CapabilityState( + str(expectation.require_state).strip().upper() + ) + except ValueError: + continue + requirements.append( + RequiredCapability(capability=capability, require_state=require_state) + ) + + expected_agent = None + if release is not None: + expected_agent = AgentIdentity( + agent_id=release.agent_id, + agent_version=release.version, + adapter_version=release.adapter.version if release.adapter else None, + ) + + request = CompatibilityRequest( + receipt=receipt, + requirements=requirements, + expected_environment=EnvironmentTuple( + os=payload.environment.os, + arch=payload.environment.arch, + ide_build=payload.environment.ide_build, + plugin_version=payload.environment.plugin_version, + host_kind=payload.environment.host_kind, + ), + expected_agent=expected_agent, + expected_protocol_version=( + protocol.environment_requirements.expected_protocol_version + ), + ) + return evaluate_compatibility(request), receipt_ref + + +def _compatibility_store(): + """Late import so the core research packages stay backend-agnostic.""" + from research.compatibility import store as compatibility_store + + return compatibility_store + + +def _agent_profile_projection( + db: Any, condition: Any +) -> Optional[BootstrapAgentProfile]: + """Project a condition's frozen config into the secret-free manifest. + + Only non-secret provider/model identity is read, and only from the config + frozen into the published revision: a later edit to the profile template must + not change a published study's manifest. The connection endpoint and secret + reference are deliberately absent — inference is relayed through the backend, + which resolves the secret at request time. + """ + resolved = getattr(condition, "resolved_distribution", None) + config = getattr(resolved, "agent_config", None) if resolved is not None else None + if config is None: + return None + return BootstrapAgentProfile( + profile_id=config.profile_id, + name=config.name, + framework_version=config.framework_version or "", + model=config.model, + # Provider endpoint stays server-side; the runtime never needs it. + base_url=None, + temperature=config.temperature, + ) + + +class _PersistentSessionFactory: + """Bootstrap session factory that persists the authoritative session row. + + Bootstrap is the first server-side component to observe an eligible + enrollment, so it is where the durable research session is created. The + ``research_session_id`` embedded in the manifest must resolve through + :func:`research.runtime.sessions.store.get_session`; otherwise ingestion rejects + every uploaded event as ``SESSION_OUT_OF_SCOPE``. Sessions are scoped to an + opaque execution context: repeated bootstrap for the same ``(enrollment, + context_id)`` reuses that context's non-terminal session, while different + windows/projects get distinct sessions sharing the enrollment/assignment. + """ + + def __init__( + self, + db: Any, + manifest_digest_provider: Optional[Callable[[], str]] = None, + *, + commit: bool = True, + ) -> None: + self._db = db + self._manifest_digest_provider = manifest_digest_provider + # Bootstrap passes commit=False so the assignment and session commit as + # one unit of work. + self._commit = commit + + def _manifest_digest(self) -> str: + if self._manifest_digest_provider is None: + return "" + return self._manifest_digest_provider() + + def create_for_enrollment( + self, + enrollment: Any, + revision: Any, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: + existing = session_store.get_active_session_for_context( + self._db, enrollment.enrollment_id, context_id + ) + if existing is not None: + return ResearchSessionRef( + research_session_id=existing.session_id, + opened_at=existing.opened_at or now, + ) + + session = ResearchSessionV1( + research_session_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_revision_id=revision.revision_id, + context_id=context_id, + state=SessionState.NOT_STARTED, + opened_at=now, + manifest_digest=self._manifest_digest(), + ) + row = session_store.create_session(self._db, session, commit=self._commit) + return ResearchSessionRef( + research_session_id=row.session_id, + opened_at=row.opened_at or now, + ) + + +@router.post( + "/research-sessions", + summary="Issue a signed bootstrap manifest for an active enrollment", +) +def create_research_session( + payload: ResearchSessionRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Validate eligibility/compatibility, then sign a secret-free manifest.""" + signer = _require_signer() + db = app.get_db_session() + try: + enrollment = _owned_enrollment(db, current_user, payload.enrollment_id) + + revision_row = protocol_store.get_revision(db, enrollment.study_revision_id) + if revision_row is None: + raise HTTPException(status_code=404, detail="Study revision not found") + revision = protocol_store.row_to_revision(revision_row) + protocol = StudyProtocolV1.model_validate(revision.protocol_json) + + assignment_row = assignment_store.get_assignment_for_enrollment_revision( + db, enrollment.enrollment_id, revision.revision_id + ) + existing = ( + assignment_store.row_to_assignment(assignment_row) + if assignment_row is not None + else None + ) + allocation = allocate( + enrollment, + revision, + existing=existing, + rng=random.Random(), + now=_now(), + ) + if allocation.outcome not in ( + AllocationOutcome.CREATED, + AllocationOutcome.EXISTING, + ): + raise HTTPException( + status_code=409, detail=_issue_payload(allocation.issue) + ) + assert allocation.assignment is not None + + condition = next( + ( + candidate + for candidate in protocol.conditions + if candidate.condition_id == allocation.assignment.condition_id + ), + None, + ) + release = None + if condition is not None: + # Read the release pin from the revision's frozen distribution, never + # from the mutable profile row (a later profile edit must not change + # a published study's manifest). + resolved_pin = getattr(condition, "resolved_distribution", None) + release_id = getattr(resolved_pin, "release_id", None) + if release_id: + release_row = registry_store.get_release(db, release_id) + if release_row is not None: + release = registry_store.row_to_release(release_row) + + compatibility_result, receipt_ref = _evaluate_compatibility( + payload, protocol, release, db + ) + + # Single unit of work (lock order: participant/enrollment already + # resolved by the caller → sticky assignment → this context's execution + # session). The session factory is commit=False, so every row created + # here lands only in the one commit below; any failure rolls back. + result = compose_bootstrap( + enrollment, + revision, + allocation.assignment, + release, + receipt_ref, + _PersistentSessionFactory(db, commit=False), + signer, + now=_now(), + compatibility_result=compatibility_result, + platform=(payload.environment.os, payload.environment.arch), + agent_profile=_agent_profile_projection(db, condition), + kill_switch_check=_kill_switch_for_scope( + db, + study_id=revision.study_id, + revision_id=revision.revision_id, + enrollment_id=enrollment.enrollment_id, + ), + context_id=payload.context_id, + ) + if result.manifest is None: + # Nothing is committed yet: a failure here (kill switch, incompatible + # environment, unresolved release) leaves no orphan session or + # half-assignment. + db.rollback() + raise HTTPException( + status_code=409, detail=_issue_payload(result.issue) + ) + + manifest = result.manifest + # Persist the sticky assignment only after the manifest is actually + # issued, still inside the same transaction. A concurrent first use that + # won the unique constraint is a retryable conflict (nothing committed). + if allocation.created: + winner_row = assignment_store.create_assignment( + db, allocation.assignment, commit=False + ) + if ( + winner_row is not None + and winner_row.assignment_id != allocation.assignment.assignment_id + ): + db.rollback() + raise HTTPException( + status_code=409, + detail={ + "code": "ASSIGNMENT_RACE", + "message": ( + "the sticky assignment was allocated concurrently; " + "retry to adopt the existing condition" + ), + }, + ) + # Single commit for assignment + session. + db.commit() + + # The session capability is verified statelessly from its HMAC signature + # and the enrollment's revocation epoch, so it is never persisted. + return JsonResponseWithStatus( + status_code=201, + content={ + "manifest": manifest.model_dump(mode="json"), + "manifest_digest": manifest.manifest_digest, + "signature": manifest.signature, + }, + ) + finally: + db.close() + + +@router.post("/exposures", summary="Record an idempotent exposure receipt") +def create_exposure( + payload: ExposureRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Record (or idempotently replay) an exposure or launch-failure receipt.""" + now = _now() + db = app.get_db_session() + try: + enrollment = _owned_enrollment(db, current_user, payload.enrollment_id) + + assignment_row = assignment_store.get_assignment(db, payload.assignment_id) + if assignment_row is None: + raise HTTPException(status_code=404, detail="Assignment not found") + assignment = assignment_store.row_to_assignment(assignment_row) + if assignment.enrollment_id != enrollment.enrollment_id: + raise HTTPException(status_code=404, detail="Assignment not found") + + # The exposure must be authorized by a capability bound to this exact + # enrollment (and revision): a capability for another participant can + # never be replayed to record an exposure here. + _verify_exposure_capability( + payload.capability, enrollment=enrollment, assignment=assignment, now=now + ) + + existing_row = assignment_store.get_exposure_by_idempotency_key( + db, payload.idempotency_key + ) + existing = ( + assignment_store.row_to_exposure(existing_row) + if existing_row is not None + else None + ) + + result = record_exposure( + assignment, + payload.environment, + payload.outcome, + payload.idempotency_key, + existing=existing, + agent_release_id=payload.agent_release_id, + artifact_digest=payload.artifact_digest, + adapter_version=payload.adapter_version, + observed_configuration=payload.observed_configuration, + evidence_digest=payload.evidence_digest, + now=now, + kill_switch_check=_kill_switch_for_scope( + db, + study_id=enrollment.study_id, + revision_id=assignment.study_revision_id, + enrollment_id=enrollment.enrollment_id, + ), + ) + if not result.accepted: + raise HTTPException(status_code=409, detail=_issue_payload(result.issue)) + + receipt = None + if result.exposure is not None and not result.reused: + row = assignment_store.insert_exposure(db, result.exposure) + receipt = assignment_store.exposure_summary(row) + + return JsonResponseWithStatus( + status_code=200 if result.reused else 201, + content={ + "accepted": True, + "is_exposure": result.is_exposure, + "reused": result.reused, + "audit": result.audit, + "exposure": receipt, + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/join.py b/src/backend/routers/research/join.py new file mode 100644 index 0000000..9e30c78 --- /dev/null +++ b/src/backend/routers/research/join.py @@ -0,0 +1,195 @@ +"""Participant self-enrollment by study join code. + +Mounted under ``/api/research/join``: + +* ``GET /{join_code}`` resolves a shared code to study/revision metadata and the + single global consent text for any authenticated user. It never returns + account, participant or secret material. +* ``POST ""`` redeems the code for the *caller*: it creates (or reuses) their + participant mapping and ACTIVE enrollment for the code's exact revision, and + returns the enrollment identity. Consent is a single acceptance recorded once + here (``consent_accepted_at``); there is no document identity, re-consent or + withdrawal. Posting is idempotent: a repeat returns the same enrollment and + never mints a second one. + +The request can never choose the revision or the eligibility verdict: the +revision comes from the join code and is always derived server-side. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, +) +from research.participants import identity as identity_store +from research.study.protocol import store as protocol_store +from research.study.protocol.enums import RevisionStatus + +if TYPE_CHECKING: + from research.participants.models import IdentityIssue + +router = APIRouter() + +# The one global consent policy. There is deliberately no per-study document +# identity/version/digest and no consent table: a participant accepts this text +# once when they join, and that acceptance is recorded on the enrollment. +GLOBAL_CONSENT_TEXT = ( + "This study collects metadata about how you use the research agent and the " + "IDE (for example which tools run, timings, token usage and file-level " + "activity). Prompts, source code and tool output are collected only if the " + "study's published telemetry policy allows content. Provider credentials are " + "never collected. Your study-local pseudonym is used in all research data; " + "your account identity is stored only in the private enrollment mapping." +) + + +class JoinRequestBody(BaseModel): + """Redeem a join code for the authenticated account. + + ``extra="forbid"`` is deliberate: a client cannot smuggle in a different + ``revision_id``, an ``eligibility`` verdict or a consent document; those are + always derived server-side from the code's published revision. + """ + + model_config = ConfigDict(extra="forbid") + + join_code: str = Field(min_length=1) + accept_consent: bool = False + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue_payload(issue: Optional[IdentityIssue]) -> dict[str, Any]: + if issue is None: + return {} + return issue.model_dump(mode="json") + + +def _join_payload(enrollment: Any, *, created: bool, reused: bool) -> dict[str, Any]: + return { + "enrollment_id": str(enrollment.enrollment_id), + "study_id": str(enrollment.study_id), + "revision_id": str(enrollment.study_revision_id), + "status": enrollment.status.value, + "created": created, + "reused": reused, + } + + +def _load_revision_by_code(db: Any, join_code: str): + """Return ``(row, domain revision)`` for a stored code, or ``None``.""" + row = protocol_store.get_revision_by_join_code(db, join_code) + if row is None: + return None + return row, protocol_store.row_to_revision(row) + + +@router.get( + "/{join_code}", summary="Resolve a study join code to revision metadata" +) +def resolve_join_code( + join_code: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Any authenticated user: study/revision metadata for a shared code. + + Only non-secret study metadata, the revision identity/digest and the global + consent text are returned; no account or participant data. + """ + db = app.get_db_session() + try: + loaded = _load_revision_by_code(db, join_code) + if loaded is None: + raise HTTPException(status_code=404, detail="Join code not found") + row, revision = loaded + study = protocol_store.get_study(db, revision.study_id) + published_at = revision.published_at + return JsonResponseWithStatus( + status_code=200, + content={ + "join_code": row.join_code, + "study": { + "study_id": str(revision.study_id), + "name": study.name if study is not None else "", + }, + "revision": { + "revision_id": str(revision.revision_id), + "revision_number": revision.revision_number, + "status": revision.status.value, + "protocol_digest": revision.protocol_digest, + "published_at": ( + published_at.isoformat() + if isinstance(published_at, datetime) + else None + ), + }, + "consent": {"text": GLOBAL_CONSENT_TEXT}, + }, + ) + finally: + db.close() + + +@router.post("", summary="Redeem a join code and enroll the caller") +def redeem_join_code( + payload: JoinRequestBody, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Idempotently enroll the caller in the code's published revision.""" + now = _now() + db = app.get_db_session() + try: + loaded = _load_revision_by_code(db, payload.join_code) + if loaded is None: + raise HTTPException(status_code=404, detail="Join code not found") + _row, revision = loaded + if revision.status != RevisionStatus.PUBLISHED: + raise HTTPException( + status_code=409, + detail={ + "code": "UNKNOWN_REVISION", + "message": "the join code's revision is not published", + }, + ) + + if not payload.accept_consent: + raise HTTPException( + status_code=409, + detail={ + "code": "CONSENT_REQUIRED", + "message": "joining a study requires accepting the consent text", + }, + ) + + ref = identity_store.revision_ref_from_study_revision(revision) + + # Shared account-wide enrollment owner: locks the participant, enforces + # one live enrollment and refuses a rejoin to a completed study. + opened = identity_store.open_enrollment(db, current_user.user_id, ref, now=now) + if opened.issue is not None or opened.enrollment is None: + raise HTTPException(status_code=409, detail=_issue_payload(opened.issue)) + + return JsonResponseWithStatus( + status_code=201 if opened.created else 200, + content=_join_payload( + opened.enrollment, + created=opened.created, + reused=not opened.created, + ), + ) + finally: + db.close() diff --git a/src/backend/routers/research/operations.py b/src/backend/routers/research/operations.py new file mode 100644 index 0000000..b00bd7c --- /dev/null +++ b/src/backend/routers/research/operations.py @@ -0,0 +1,122 @@ +"""Kill-switch API (Issue 13). + +These admin-only routes are merged with the Issue 12 read-model routes behind +the single ``/research/operations`` router (see +:mod:`backend.routers.research`). All handlers are thin; the domain logic lives +in :mod:`research.analysis.operations`. Every endpoint here requires an +operator/admin identity because it engages/releases an enforcement control. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin +from research.analysis.operations import store as operations_store +from research.analysis.operations.enums import ( + KillSwitchScopeKind, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.analysis.operations.models import ( + KillSwitchRecord, + KillSwitchScope, +) + +operations_router = APIRouter() + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +# --------------------------------------------------------------------------- +# Request bodies +# --------------------------------------------------------------------------- + + +class KillSwitchEngageRequest(BaseModel): + """Engage a kill switch at a study/revision/enrollment scope.""" + + scope_kind: KillSwitchScopeKind + scope_id: uuid.UUID + reason: str + effective_until: Optional[datetime] = None + + +@operations_router.get("/kill-switch", summary="List kill-switch records") +def list_kill_switches( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + records = operations_store.kill_switch_records(db) + return JsonResponseWithStatus( + status_code=200, + content={ + "records": [record.model_dump(mode="json") for record in records], + "engaged": [ + str(record.switch_id) + for record in records + if record.is_engaged(_now()) + ], + }, + ) + finally: + db.close() + + +@operations_router.post("/kill-switch", summary="Engage a kill switch") +def engage_kill_switch( + payload: KillSwitchEngageRequest, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + record = KillSwitchRecord( + switch_id=uuid.uuid4(), + scope=KillSwitchScope( + kind=payload.scope_kind, scope_id=payload.scope_id + ), + reason=payload.reason, + actor=current_user.email, + engaged_at=_now(), + effective_until=payload.effective_until, + ) + row = operations_store.engage_kill_switch(db, record) + return JsonResponseWithStatus( + status_code=201, content={"switch_id": str(row.switch_id), "reason": row.reason} + ) + finally: + db.close() + + +@operations_router.post( + "/kill-switch/{switch_id}/release", summary="Release a kill switch" +) +def release_kill_switch_endpoint( + switch_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + row = operations_store.release_kill_switch(db, switch_id, released_at=_now()) + if row is None: + raise HTTPException(status_code=404, detail="Kill switch not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "switch_id": str(row.switch_id), + "released_at": row.released_at.isoformat() if row.released_at else None, + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/packages.py b/src/backend/routers/research/packages.py new file mode 100644 index 0000000..28f2296 --- /dev/null +++ b/src/backend/routers/research/packages.py @@ -0,0 +1,202 @@ +"""Runtime package and conformance API (Issue 11). + +Mounted under ``/research/packages``. All endpoints are admin-only: they record +artifact/manifest evidence and can promote an agent release's qualification, so +they are never exposed to participants. Handlers are thin; the packaging domain +logic lives in :mod:`research.study.packaging`. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin +from research.study.agents import store as agents_store +from research.study.packaging import store as packaging_store +from research.study.packaging.conformance import qualification_for_release +from research.study.packaging.models import ( + ConformanceReceiptV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + RuntimeManifestV2, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) + +packages_router = APIRouter() + + +class QualificationLinkRequest(BaseModel): + """Ask the registry to link conformance evidence to a release qualification.""" + + release_id: str + os: str + arch: str + required_cases: list[str] = Field(default_factory=list) + + +@packages_router.get("", summary="List registered runtime packages") +def list_packages( + release_id: str | None = None, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + rows = packaging_store.list_packages(db, release_id) + return JsonResponseWithStatus( + status_code=200, + content={"packages": [packaging_store.package_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@packages_router.post("", summary="Register a runtime package manifest") +def register_package( + payload: RuntimeManifestV2, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + if not payload.digest_matches(): + raise HTTPException( + status_code=422, + detail="manifest_digest does not match the canonical manifest content", + ) + row = packaging_store.upsert_package(db, payload) + return JsonResponseWithStatus( + status_code=201, content=packaging_store.package_summary(row) + ) + finally: + db.close() + + +@packages_router.get("/receipts", summary="List conformance receipts") +def list_receipts( + artifact_digest: str | None = None, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + rows = packaging_store.list_receipts(db, artifact_digest) + return JsonResponseWithStatus( + status_code=200, + content={"receipts": [row.model_dump(mode="json") for row in rows]}, + ) + finally: + db.close() + + +@packages_router.post("/receipts", summary="Record a conformance receipt") +def create_receipt( + payload: ConformanceReceiptV1, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + try: + row = packaging_store.insert_receipt(db, payload) + except LookupError as error: + raise HTTPException(status_code=409, detail=str(error)) from error + return JsonResponseWithStatus( + status_code=201, + content={ + "receipt_id": str(row.receipt_id), + "status": row.status.value, + "host": row.host.key, + "artifact_digest": row.artifact_digest, + }, + ) + finally: + db.close() + + +@packages_router.get("/coverage", summary="Conformance coverage view") +def conformance_coverage( + release_id: str | None = None, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + return JsonResponseWithStatus( + status_code=200, + content=packaging_store.conformance_coverage(db, release_id), + ) + finally: + db.close() + + +@packages_router.post( + "/{package_id}/qualification", summary="Report derived qualification for a package" +) +def link_qualification( + package_id: uuid.UUID, + payload: QualificationLinkRequest, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + """Report whether a receipt proves all required cases PASS. + + Qualification is derived from the stored conformance evidence, so this + endpoint no longer mutates a release (a caller can never mark a release + QUALIFIED). Recording a passing receipt is the only promotion path; this + handler evaluates the requested platform/case set against the already-stored + receipts and reports both the decision and the release's derived status. + """ + db = app.get_db_session() + try: + package_row = packaging_store.get_package(db, package_id) + if package_row is None: + raise HTTPException(status_code=404, detail="Runtime package not found") + release_row = agents_store.get_release(db, payload.release_id) + if release_row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + release = agents_store.row_to_release(release_row) + receipts = [ + packaging_store.row_to_receipt(row) + for row in packaging_store.list_receipts(db) + ] + decision = qualification_for_release( + release, + receipts, + required_cases=payload.required_cases, + os=payload.os, + arch=payload.arch, + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "promote": decision.promote, + "reason": decision.reason.value, + "message": decision.message, + "receipt_id": str(decision.receipt_id) if decision.receipt_id else None, + "release_id": payload.release_id, + "qualification_status": release.qualification_status.value, + }, + ) + finally: + db.close() + + +@packages_router.get("/{package_id}", summary="Get a runtime package") +def get_package( + package_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + row = packaging_store.get_package(db, package_id) + if row is None: + raise HTTPException(status_code=404, detail="Runtime package not found") + return JsonResponseWithStatus( + status_code=200, content=packaging_store.package_summary(row) + ) + finally: + db.close() diff --git a/src/backend/routers/research/participants.py b/src/backend/routers/research/participants.py new file mode 100644 index 0000000..490638d --- /dev/null +++ b/src/backend/routers/research/participants.py @@ -0,0 +1,279 @@ +"""Participant enrollment inspection and privacy-operator retention views. + +Mounted under ``/api/research/participants``. Consent is a single acceptance +recorded at join; there is no consent, re-consent or withdrawal surface here. + +* ``GET /me`` — the caller's study-local enrollment projections. +* ``POST /enrollments`` — create (or reuse) the caller's enrollment for a + published revision; delegates to the same account-wide owner as the join code. +* ``GET /enrollments/{id}`` — administrator inspection. +* deletion-ledger / retention-job routes — administrator privacy-operator views + for the study-end retention path. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from research.participants import identity as store +from research.participants import retention as ret_api +from research.participants.enums import RetentionJobState +from research.participants.models import RevisionRef +from research.study.protocol import store as protocol_store +from research.study.protocol.enums import RevisionStatus + +router = APIRouter() + + +class EnrollmentRequestBody(BaseModel): + """Request enrollment for an existing published revision.""" + + study_id: uuid.UUID + revision_id: uuid.UUID + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue_payload(issue: Any) -> dict[str, Any]: + if issue is None: + return {} + return issue.model_dump(mode="json") + + +def _load_revision_ref(db: Any, revision_id: uuid.UUID) -> Optional[RevisionRef]: + """Return the published revision's enrollment slice, or ``None``. + + Both entry points (join and participants) must bind enrollment to a + *published* revision; a draft or retired revision never enrolls. + """ + row = protocol_store.get_revision(db, revision_id) + if row is None: + return None + if row.status != RevisionStatus.PUBLISHED.value: + return None + return store.revision_ref_from_mapping( + row.protocol_json, revision_id=row.revision_id, study_id=row.study_id + ) + + +def _owned_enrollment( + db: Any, current_user: AuthenticatedUser, enrollment_id: uuid.UUID +): + """Return the caller's enrollment or raise 404 without revealing ownership.""" + participant_row = store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + row = store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + enrollment = store.row_to_enrollment(row) + if enrollment.participant_id != participant_row.participant_id: + raise HTTPException(status_code=404, detail="Enrollment not found") + return enrollment + + +@router.get("/me", summary="Get my study enrollment status") +def get_my_status( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return only study-local enrollment projections for the caller.""" + db = app.get_db_session() + try: + participant_row = store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + return JsonResponseWithStatus(status_code=200, content={"enrollments": []}) + rows = store.list_enrollments(db, participant_row.participant_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "enrollments": [ + store.researcher_projection(store.row_to_enrollment(row)) + for row in rows + ] + }, + ) + finally: + db.close() + + +@router.post("/enrollments", summary="Request enrollment in a study revision") +def request_enrollment( + payload: EnrollmentRequestBody, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Create (or reuse) the caller's enrollment; never a second active one.""" + db = app.get_db_session() + try: + revision_ref = _load_revision_ref(db, payload.revision_id) + if revision_ref is None: + raise HTTPException( + status_code=404, + detail={ + "code": "REVISION_NOT_PUBLISHED", + "message": "study revision not found or not published", + }, + ) + if revision_ref.study_id != payload.study_id: + raise HTTPException( + status_code=409, + detail={ + "code": "REVISION_STUDY_MISMATCH", + "message": "the revision does not belong to the given study", + }, + ) + + # Same shared account-wide enrollment owner as the join-code route. + opened = store.open_enrollment(db, current_user.user_id, revision_ref, now=_now()) + if opened.issue is not None or opened.enrollment is None: + raise HTTPException(status_code=409, detail=_issue_payload(opened.issue)) + + return JsonResponseWithStatus( + status_code=201 if opened.created else 200, + content={ + "enrollment": store.researcher_projection(opened.enrollment), + "created": opened.created, + }, + ) + finally: + db.close() + + +@router.get("/enrollments/{enrollment_id}", summary="Inspect an enrollment") +def get_enrollment_route( + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Administrator view of one pseudonymous enrollment projection.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + return JsonResponseWithStatus( + status_code=200, + content={"enrollment": store.researcher_projection(store.row_to_enrollment(row))}, + ) + finally: + db.close() + + +@router.get("/deletion-ledger", summary="Read the deletion ledger") +def read_deletion_ledger( + enrollment_id: Optional[uuid.UUID] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of the append-only deletion ledger.""" + require_admin(current_user) + db = app.get_db_session() + try: + rows = store.list_deletion_ledger(db, enrollment_id) + return JsonResponseWithStatus( + status_code=200, + content={"entries": [store.deletion_ledger_summary(row) for row in rows]}, + ) + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Retention execution jobs (privacy-operator) +# --------------------------------------------------------------------------- + +_RETRYABLE_STATES = frozenset({RetentionJobState.FAILED, RetentionJobState.RETRYABLE}) + + +@router.get("/retention-jobs/{job_id}", summary="Fetch one retention job's status") +def get_retention_job_status( + job_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of one content-free retention job summary.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = ret_api.get_retention_job(db, job_id) + if row is None: + raise HTTPException(status_code=404, detail="Retention job not found") + return JsonResponseWithStatus( + status_code=200, + content={"job": ret_api.job_summary(ret_api.row_to_retention_job(row))}, + ) + finally: + db.close() + + +@router.get( + "/enrollments/{enrollment_id}/retention-jobs", + summary="List an enrollment's retention jobs", +) +def list_enrollment_retention_jobs( + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of an enrollment's retention-job history.""" + require_admin(current_user) + db = app.get_db_session() + try: + rows = ret_api.list_retention_jobs_for_enrollment(db, enrollment_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "jobs": [ + ret_api.job_summary(ret_api.row_to_retention_job(row)) + for row in rows + ] + }, + ) + finally: + db.close() + + +@router.post( + "/retention-jobs/{job_id}/retry", + summary="Re-drive a FAILED/RETRYABLE retention job", +) +def retry_retention_job( + job_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Re-drive a failed/retryable job; a completed job is never re-executed.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = ret_api.get_retention_job(db, job_id) + if row is None: + raise HTTPException(status_code=404, detail="Retention job not found") + job = ret_api.row_to_retention_job(row) + if job.state not in _RETRYABLE_STATES: + raise HTTPException( + status_code=409, + detail={"code": "NOT_RETRYABLE", "state": job.state.value}, + ) + redriven = ret_api.run_retention_job(db, job_id, now=_now()) + return JsonResponseWithStatus( + status_code=200, content={"job": ret_api.job_summary(redriven)} + ) + finally: + db.close() diff --git a/src/backend/routers/research/providers.py b/src/backend/routers/research/providers.py new file mode 100644 index 0000000..6d773be --- /dev/null +++ b/src/backend/routers/research/providers.py @@ -0,0 +1,238 @@ +"""Administrator-managed provider connections and researcher grants. + +Mounted under ``/api/research``. A ``provider_connection`` owns the upstream +``base_url`` and the *name* of the deployment secret (``secret_ref``). Responses +never include the secret value; the env-var name and endpoint URL are exposed to +administrators only. Researchers see only the connections explicitly granted to +them, identified by id/label with their allowed models and a readiness flag. +""" + +from __future__ import annotations + +import json +import os +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.access import require_researcher +from database import crud + +router = APIRouter() + + +def _parse_models(models_json: Any) -> list[str]: + try: + parsed = json.loads(models_json or "[]") + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(parsed, list): + return [] + return [str(item) for item in parsed] + + +def _secret_present(secret_ref: Optional[str]) -> bool: + """Whether the deployment environment currently holds the secret value.""" + name = (secret_ref or "").strip() + return bool(name) and bool(os.getenv(name, "").strip()) + + +def _safe_payload(connection: Any, *, admin: bool) -> dict[str, Any]: + """Never include the secret value. Endpoint/ref name are admin-only.""" + payload: dict[str, Any] = { + "connection_id": str(connection.connection_id), + "label": connection.label, + "models": _parse_models(connection.models_json), + "is_active": bool(connection.is_active), + # Readiness is derived, never a stored credential. + "ready": _secret_present(connection.secret_ref) and bool(connection.is_active), + } + if admin: + payload["base_url"] = connection.base_url + # Name of the environment variable only — never its value. + payload["secret_ref"] = connection.secret_ref + return payload + + +class ProviderConnectionPayload(BaseModel): + """Admin-maintained connection; ``extra=forbid`` blocks typo'd fields.""" + + model_config = ConfigDict(extra="forbid") + + label: str = Field(..., min_length=1) + base_url: str = Field(..., min_length=1) + # Name of the deployment env var holding the key, never the key itself. + secret_ref: str = Field(..., min_length=1) + models: list[str] = Field(default_factory=list) + is_active: bool = True + + @field_validator("label", "base_url", "secret_ref") + @classmethod + def normalize_text(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("value must not be blank") + return normalized + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: str) -> str: + if not (value.startswith("http://") or value.startswith("https://")): + raise ValueError("base_url must be an http(s) endpoint") + if any(ch in value for ch in ("#",)) or "@" in value.split("//", 1)[-1]: + raise ValueError("base_url must not contain credentials or a fragment") + return value.rstrip("/") + + @field_validator("secret_ref") + @classmethod + def validate_secret_ref(cls, value: str) -> str: + """The field is an env-var *name*, never a pasted credential.""" + import re + + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,127}", value): + raise ValueError( + "secret_ref must be the NAME of an environment variable " + "(e.g. EXAMPLE_PROVIDER_KEY), not a key value" + ) + return value + + @field_validator("models") + @classmethod + def validate_models(cls, value: list[str]) -> list[str]: + cleaned = [str(item).strip() for item in value if str(item).strip()] + if not cleaned: + raise ValueError("at least one allowed model is required") + return sorted(set(cleaned)) + + +def _connection_payload_json(payload: ProviderConnectionPayload) -> str: + return json.dumps(payload.models) + + +@router.get( + "/provider-connections", + summary="List provider connections available to the caller", +) +def list_provider_connections( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Administrators see all connections; researchers see only granted ones.""" + require_researcher(current_user) + db = app.get_db_session() + try: + if current_user.is_admin: + connections = crud.list_provider_connections(db) + else: + connections = crud.list_available_provider_connections( + db, current_user.user_id + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "connections": [ + _safe_payload(connection, admin=current_user.is_admin) + for connection in connections + ] + }, + ) + finally: + db.close() + + +@router.post( + "/provider-connections", + summary="Create a provider connection (admin)", +) +def create_provider_connection( + payload: ProviderConnectionPayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + if crud.get_provider_connection_by_label(db, payload.label) is not None: + raise HTTPException( + status_code=409, detail="A connection with that label already exists" + ) + connection = crud.create_provider_connection( + db, + label=payload.label, + base_url=payload.base_url, + secret_ref=payload.secret_ref, + models_json=_connection_payload_json(payload), + is_active=payload.is_active, + ) + return JsonResponseWithStatus( + status_code=201, + content={"connection": _safe_payload(connection, admin=True)}, + ) + finally: + db.close() + + +@router.put( + "/provider-connections/{connection_id}", + summary="Update a provider connection (admin)", +) +def update_provider_connection( + connection_id: uuid.UUID, + payload: ProviderConnectionPayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + connection = crud.update_provider_connection( + db, + connection_id, + label=payload.label, + base_url=payload.base_url, + secret_ref=payload.secret_ref, + models_json=_connection_payload_json(payload), + is_active=payload.is_active, + ) + if connection is None: + raise HTTPException(status_code=404, detail="Connection not found") + return JsonResponseWithStatus( + status_code=200, + content={"connection": _safe_payload(connection, admin=True)}, + ) + finally: + db.close() + + +@router.delete( + "/provider-connections/{connection_id}", + summary="Delete a provider connection (admin)", +) +def delete_provider_connection( + connection_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + deleted = crud.delete_provider_connection(db, connection_id) + if not deleted: + raise HTTPException(status_code=404, detail="Connection not found") + return JsonResponseWithStatus( + status_code=200, + content={"deleted": True, "connection_id": str(connection_id)}, + ) + finally: + db.close() + + diff --git a/src/backend/routers/research/read_models.py b/src/backend/routers/research/read_models.py new file mode 100644 index 0000000..ce6c468 --- /dev/null +++ b/src/backend/routers/research/read_models.py @@ -0,0 +1,112 @@ +"""Researcher operations read models (Issue 12). + +``operations_router`` - RBAC-scoped revision, enrollment, exposure and +telemetry coverage/metric read models. It is contributed to the single +``/operations`` router (see :mod:`backend.routers.research`) rather than +mounted separately, so there is exactly one operations/read API surface. +Handler logic lives in :mod:`research.analysis.read_models`. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, +) +from research.analysis.read_models import store as read_store +from research.analysis.read_models.service import ( + build_condition_exposures, + build_enrollment_coverage, +) +from research.study.protocol import store as protocol_store + +operations_router = APIRouter() + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _authorize(db: Any, current_user: AuthenticatedUser, study_id: uuid.UUID) -> None: + """Authorize study read access by ownership (admin bypasses).""" + from backend.routers.research.access import require_study_owner + + study = protocol_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + + +def _load_revision(db: Any, revision_id: uuid.UUID, study_id: uuid.UUID): + row = protocol_store.get_revision(db, revision_id) + if row is None: + raise HTTPException(status_code=404, detail="Study revision not found") + revision = protocol_store.row_to_revision(row) + if revision.study_id != study_id: + raise HTTPException(status_code=409, detail="revision does not belong to study") + return revision + + +@operations_router.get( + "/enrollments/coverage", summary="Enrollment-state coverage for a revision" +) +def enrollment_coverage( + study_id: uuid.UUID, + revision_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + revision = _load_revision(db, revision_id, study_id) + enrollments = read_store.list_enrollments(db, study_id) + model = build_enrollment_coverage( + enrollments, + study_id=study_id, + revision_id=revision_id, + revision_digest=revision.protocol_digest, + ) + return JsonResponseWithStatus(status_code=200, content=model.model_dump(mode="json")) + finally: + db.close() + + +@operations_router.get( + "/exposures", summary="Assignment vs exposure per condition" +) +def condition_exposures( + study_id: uuid.UUID, + revision_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + revision = _load_revision(db, revision_id, study_id) + assignments = read_store.list_assignments(db, study_id) + exposures = read_store.list_exposures(db, revision_id) + models = build_condition_exposures( + assignments, + exposures, + study_id=study_id, + revision_id=revision_id, + revision_digest=revision.protocol_digest, + ) + return JsonResponseWithStatus( + status_code=200, + content={"conditions": [model.model_dump(mode="json") for model in models]}, + ) + finally: + db.close() + + diff --git a/src/backend/routers/research/researchers.py b/src/backend/routers/research/researchers.py new file mode 100644 index 0000000..1362c57 --- /dev/null +++ b/src/backend/routers/research/researchers.py @@ -0,0 +1,76 @@ +"""Administrator researcher enablement (P3). + +Mounted under ``/api/research/researchers``. An administrator flips the single +``user.can_research`` flag that lets an account own private profiles and create +studies. There is no self-service route: no non-administrator can change +``can_research`` or ``is_admin``. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + require_admin, +) +from database import crud + +router = APIRouter() + + +class ResearcherEnableRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + can_research: bool + + +def _user_payload(user) -> dict: + return { + "user_id": str(user.user_id), + "email": user.email, + "name": user.name, + "is_admin": bool(user.is_admin), + "can_research": bool(user.can_research), + } + + +@router.get("", summary="List enabled researcher accounts (admin)") +def list_researchers( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + return JsonResponseWithStatus( + status_code=200, + content={"researchers": [ _user_payload(u) for u in crud.list_researchers(db)]}, + ) + finally: + db.close() + + +@router.put("/{user_id}", summary="Enable or disable a researcher account (admin)") +def set_researcher_enabled( + user_id: uuid.UUID, + payload: ResearcherEnableRequest, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + user = crud.set_user_can_research(db, user_id, payload.can_research) + if user is None: + raise HTTPException(status_code=404, detail="User not found") + return JsonResponseWithStatus( + status_code=200, content={"user": _user_payload(user)} + ) + finally: + db.close() diff --git a/src/backend/routers/research/sessions.py b/src/backend/routers/research/sessions.py new file mode 100644 index 0000000..973dc72 --- /dev/null +++ b/src/backend/routers/research/sessions.py @@ -0,0 +1,486 @@ +"""Research session lifecycle API (Issue 07). + +Mounted under ``/api/research/sessions``. Endpoints are authenticated by the +short-lived, scoped session capability issued at bootstrap +(:func:`research.runtime.bootstrap.capability.verify_capability`); the enrollment's +current revocation epoch is rechecked on every call. Handlers are thin: the +state machine lives in :mod:`research.runtime.sessions.service` and persistence in +:mod:`research.runtime.sessions.store`. + +Server time is authoritative for idle/resume/expiry decisions; a client clock is +never trusted. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.research.bootstrap import BOOTSTRAP_SIGNING_SECRET +from research.analysis.operations import store as operations_store +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.models import ( + SessionCapability, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.runtime.sessions import store as session_store +from research.runtime.sessions.enums import CloseReason, SessionReasonCode, SessionState +from research.runtime.sessions.service import ( + close, + expire_if_idle, + on_qualifying_activity, + open_session, + session_policy_from_revision, +) +from research.study.protocol import store as protocol_store +from research.study.protocol.enums import RevisionStatus +from research.study.protocol.models import StudyProtocolV1 + +if TYPE_CHECKING: + from research.runtime.sessions.models import ResearchSessionV1, SessionPolicyV1 + +router = APIRouter() + +AUDIENCE = "research-runtime" +_SCOPE_WRITE = "telemetry:write" +_SCOPE_HEARTBEAT = "session:heartbeat" +_SCOPE_CLOSE = "session:close" + + +class CreateSessionRequest(BaseModel): + """Create (or reuse) a research session for an active enrollment.""" + + capability: SessionCapability + enrollment_id: uuid.UUID + study_revision_id: uuid.UUID + manifest_digest: str + # Opaque execution-context id for this project/window (never a path). + # Required: one context maps to one live session; different contexts differ. + context_id: str + environment_ref: Optional[str] = None + + +class HeartbeatRequest(BaseModel): + """Heartbeat/report-activity for an existing session.""" + + capability: SessionCapability + research_session_id: uuid.UUID + + +class CloseSessionRequest(BaseModel): + """Close a session with a typed reason.""" + + capability: SessionCapability + research_session_id: uuid.UUID + reason: CloseReason + + +class SessionSummaryRequest(BaseModel): + """Fetch the summary of an existing session.""" + + capability: SessionCapability + research_session_id: uuid.UUID + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _detail(code: str, message: str, **extra: Any) -> dict[str, Any]: + detail = {"code": code, "message": message} + detail.update(extra) + return detail + + +def _kill_switch_for_session(db: Any, session: ResearchSessionV1) -> Any: + """DB-backed kill-switch predicate scoped to this session's subject.""" + revision_row = protocol_store.get_revision(db, session.study_revision_id) + study_id = revision_row.study_id if revision_row is not None else None + return operations_store.db_kill_switch_check( + db, + study_id=study_id, + revision_id=session.study_revision_id, + enrollment_id=session.enrollment_id, + ) + + +def _authorize( + capability: SessionCapability, + enrollment: Any, + *, + scope: str, + now: datetime, + research_session_id: Optional[uuid.UUID] = None, + revision_id: Optional[uuid.UUID] = None, +) -> None: + """Verify the capability and recheck the enrollment's revocation epoch. + + ``research_session_id`` and ``revision_id`` bind the capability to the exact + session/revision being acted on, so a valid capability for another session + is rejected with a typed reason. + """ + verification = verify_capability( + capability, + BOOTSTRAP_SIGNING_SECRET or "", + expected_audience=AUDIENCE, + expected_scope=[scope], + now=now, + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=enrollment.enrollment_id, + expected_research_session_id=research_session_id, + expected_revision_id=revision_id, + ) + if not verification.ok: + raise HTTPException( + status_code=403, + detail=_detail( + SessionReasonCode.CAPABILITY_INVALID.value, + verification.message or "session capability is invalid", + capability_reason=verification.reason.value, + ), + ) + if enrollment.status != EnrollmentStatus.ACTIVE: + raise HTTPException( + status_code=403, + detail=_detail( + SessionReasonCode.ENROLLMENT_NOT_ACTIVE.value, + f"enrollment is {enrollment.status.value}; bootstrap is blocked", + ), + ) + + +def _load_enrollment(db: Any, enrollment_id: uuid.UUID): + row = identity_store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + return identity_store.row_to_enrollment(row) + + +def _load_session(db: Any, research_session_id: uuid.UUID) -> ResearchSessionV1: + row = session_store.get_session(db, research_session_id) + if row is None: + raise HTTPException(status_code=404, detail="Research session not found") + return session_store.row_to_session(row) + + +def _load_policy(db: Any, session: ResearchSessionV1) -> Optional[SessionPolicyV1]: + revision_row = protocol_store.get_revision(db, session.study_revision_id) + if revision_row is None: + return None + protocol = StudyProtocolV1.model_validate(revision_row.protocol_json) + return session_policy_from_revision(protocol) + + +def _persist_result(db: Any, result: Any) -> None: + """Persist a session result: append the transition (if any) and always + update the session row. + + Updating unconditionally is what makes an *ordinary* heartbeat durable: a + heartbeat that only refreshes ``last_activity_at`` has no transition, so + gating the row update on ``transition is not None`` would silently drop it. + """ + if result.transition is not None: + session_store.insert_transition(db, result.transition) + session_store.update_session(db, result.session) + + +def _next_actions( + session: ResearchSessionV1, policy: SessionPolicyV1, now: datetime +) -> list[str]: + if session.state == SessionState.NOT_STARTED: + return ["report_activity"] + if session.state == SessionState.RUNNING: + return ["heartbeat"] + if session.state == SessionState.OFFLINE: + return ["recover"] + if session.state == SessionState.SUSPENDED: + last = session.last_activity_at + within = ( + last is not None + and (now - last).total_seconds() <= policy.resume_grace_seconds + ) + return ["resume_within_grace"] if within else ["start_new_session"] + return [] + + +@router.post("/", summary="Create (or reuse) a research session") +def create_research_session( + payload: CreateSessionRequest, + app: App = Depends(App.get_instance), +): + """Open a session for an active enrollment, or return the active one.""" + now = _now() + db = app.get_db_session() + try: + enrollment = _load_enrollment(db, payload.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_WRITE, + now=now, + revision_id=payload.study_revision_id, + ) + + revision_row = protocol_store.get_revision(db, payload.study_revision_id) + if revision_row is None: + raise HTTPException(status_code=404, detail="Study revision not found") + if revision_row.revision_id != enrollment.study_revision_id: + raise HTTPException( + status_code=409, + detail=_detail( + "REVISION_MISMATCH", + "enrollment is not bound to the requested revision", + ), + ) + if revision_row.status != RevisionStatus.PUBLISHED.value: + raise HTTPException( + status_code=409, + detail=_detail("REVISION_NOT_PUBLISHED", "revision is not published"), + ) + + protocol = StudyProtocolV1.model_validate(revision_row.protocol_json) + policy = session_policy_from_revision(protocol) + if policy is None: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.POLICY_MISSING.value, + "revision does not declare idle/resume session policy", + ), + ) + + existing_row = session_store.get_active_session_for_context( + db, enrollment.enrollment_id, payload.context_id + ) + if existing_row is not None: + existing = session_store.row_to_session(existing_row) + return JsonResponseWithStatus( + status_code=200, + content={ + "created": False, + "session": session_store.session_summary(existing_row), + "next_actions": _next_actions(existing, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + }, + ) + + session = open_session( + enrollment, + revision_row, + manifest_digest=payload.manifest_digest, + environment_ref=payload.environment_ref, + context_id=payload.context_id, + now=now, + ) + # An engaged kill switch blocks new funded session creation. + kill_switch_check = operations_store.db_kill_switch_check( + db, + study_id=revision_row.study_id, + revision_id=payload.study_revision_id, + enrollment_id=enrollment.enrollment_id, + ) + if kill_switch_check(): + raise HTTPException( + status_code=403, + detail=_detail( + "KILL_SWITCH_ENGAGED", + "an operator kill switch is engaged for this study", + ), + ) + row = session_store.create_session(db, session) + return JsonResponseWithStatus( + status_code=201, + content={ + "created": True, + "session": session_store.session_summary(row), + "next_actions": _next_actions(session, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + }, + ) + finally: + db.close() + + +@router.post("/heartbeat", summary="Heartbeat / report session activity") +def heartbeat( + payload: HeartbeatRequest, + app: App = Depends(App.get_instance), +): + """Update last-activity (or start the session) and return next actions.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_HEARTBEAT, + now=now, + research_session_id=session.research_session_id, + revision_id=session.study_revision_id, + ) + + if session.state.is_terminal: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.SESSION_TERMINAL.value, + f"session is {session.state.value}", + ), + ) + + policy = _load_policy(db, session) + if policy is None: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.POLICY_MISSING.value, + "revision does not declare idle/resume session policy", + ), + ) + + kill_switch_check = _kill_switch_for_session(db, session) + idle_result = expire_if_idle( + session, now, policy=policy, kill_switch_check=kill_switch_check + ) + if not idle_result.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + idle_result.reason.value, + ( + idle_result.issue.message + if idle_result.issue is not None + else "heartbeat rejected" + ), + ), + ) + if idle_result.transition is not None: + _persist_result(db, idle_result) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": [], + "heartbeat_seconds": policy.heartbeat_seconds, + }, + ) + + activity = on_qualifying_activity( + idle_result.session, now, kill_switch_check=kill_switch_check + ) + if not activity.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + activity.reason.value, + ( + activity.issue.message + if activity.issue is not None + else "heartbeat rejected" + ), + ), + ) + _persist_result(db, activity) + + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": _next_actions(activity.session, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + }, + ) + finally: + db.close() + + +@router.post("/close", summary="Close a research session") +def close_research_session( + payload: CloseSessionRequest, + app: App = Depends(App.get_instance), +): + """Apply a typed terminal close reason.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_CLOSE, + now=now, + research_session_id=session.research_session_id, + revision_id=session.study_revision_id, + ) + + if session.state.is_terminal: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.SESSION_TERMINAL.value, + f"session is {session.state.value}", + ), + ) + + result = close( + session, + payload.reason, + now, + kill_switch_check=_kill_switch_for_session(db, session), + ) + if not result.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + result.reason.value, + result.issue.message if result.issue else "close rejected", + ), + ) + _persist_result(db, result) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={"session": session_store.session_summary(row), "next_actions": []}, + ) + finally: + db.close() + + +@router.post("/summary", summary="Get a research session summary") +def get_research_session( + payload: SessionSummaryRequest, + app: App = Depends(App.get_instance), +): + """Return the summary of a session, authorized by the session capability.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_WRITE, + now=now, + research_session_id=session.research_session_id, + revision_id=session.study_revision_id, + ) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={"session": session_store.session_summary(row)}, + ) + finally: + db.close() diff --git a/src/backend/routers/research/studies.py b/src/backend/routers/research/studies.py new file mode 100644 index 0000000..c8453f1 --- /dev/null +++ b/src/backend/routers/research/studies.py @@ -0,0 +1,894 @@ +"""API for versioned study protocols and immutable publication. + +Mounted under ``/api/research/studies``. The handlers are intentionally thin: +validation, canonicalization, publication and lineage live in +:mod:`research.study.protocol`, and the SQLAlchemy session is supplied by +``App.get_db_session`` and passed to the store helpers. + +Authorization is per-study RBAC rather than blanket admin: creating/validating +drafts and publishing/superseding/retiring revisions require the ``OWNER`` role +on the target study, while reading revisions/drafts is open to ``OWNER``, +``ANALYST`` and ``VIEWER``. An administrator may do everything. The study index +and the join-code read endpoints remain admin-only because they enumerate every +study on the deployment. + +Drafts are editable; published revisions are immutable and content-addressed. +Editing a published revision goes through ``/revisions/{id}/supersede`` and +always creates a successor revision. Publishing mints the study's join code. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field, field_validator + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, +) +from backend.routers.research.access import ( + is_study_owner, + require_researcher, + require_study_owner, +) +from database import crud +from research.participants import identity as identity_store +from research.study.agents import store as registry_store +from research.study.agents.distributions import resolve_distribution_view +from research.study.agents.registry import approval_option_verified +from research.study.protocol import store +from research.study.protocol.canonical import protocol_digest +from research.study.protocol.enums import PublicationOutcome +from research.study.protocol.models import ( + StudyProtocolV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.study.protocol.publication import ( + AuditRecord, + RevisionLineage, + lineage_from_revisions, + publish_revision, + retire_revision, +) +from research.study.protocol.validation import ( + DistributionResolution, + ValidationError, + blocking_errors, + scan_document_safety, + validate_protocol, + warnings, +) + +router = APIRouter() + + +class DraftCreateRequest(BaseModel): + """Create (or replace) an editable draft for a study.""" + + study_id: uuid.UUID + name: str + protocol: StudyProtocolV1 + + +class ValidateRequest(BaseModel): + """Validate a protocol document without persisting it.""" + + protocol: StudyProtocolV1 + + +class PublishRequest(BaseModel): + """Publish a protocol document as a new immutable revision.""" + + protocol: StudyProtocolV1 + # Optimistic concurrency: the caller's view of the current revision number. + expected_revision_number: Optional[int] = None + # Optional client-side canonical digest; a mismatch is rejected. + expected_protocol_digest: Optional[str] = None + + +class RetireRequest(BaseModel): + """Retire a published revision without changing its stored content.""" + + actor: Optional[str] = Field(default=None) + + +class StudyCreateRequest(BaseModel): + """Create a new study identity without pasting a pre-generated UUID.""" + + name: str = Field(..., min_length=1) + description: Optional[str] = Field(default=None) + # Defaults to the creating researcher's email when omitted. + owner: Optional[str] = Field(default=None) + # Fixed study window; a draft that omits these starts now and has no end. + starts_at: Optional[datetime] = Field(default=None) + ends_at: Optional[datetime] = Field(default=None) + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("name must not be blank") + return normalized + + +def _distribution_resolver( + db: Any, funding_owner_user_id: Any = None +) -> "_DbDistributionResolver": + """A DB-backed resolver for one request's condition distributions.""" + return _DbDistributionResolver(db, funding_owner_user_id) + + +class _DbDistributionResolver: + """Resolve a condition's ``distribution_id`` to its non-secret view. + + The distribution is an ``AgentProfile`` row; its pin is resolved against the + immutable release registry and its ``verified`` flag is *derived* from + conformance evidence. No secret is ever read here. + """ + + def __init__(self, db: Any, funding_owner_user_id: Any = None) -> None: + self._db = db + self._funding_owner_user_id = funding_owner_user_id + + def resolve(self, distribution_id: Any): + profile = crud.get_agent_profile_by_id(self._db, distribution_id) + if profile is None: + return DistributionResolution( + found=False, distribution_id=distribution_id + ) + release = None + release_id = getattr(profile, "release_id", None) + if release_id: + row = registry_store.get_release(self._db, release_id) + if row is not None: + release = registry_store.row_to_release(row) + connection = None + connection_id = getattr(profile, "connection_id", None) + if connection_id is not None: + connection = crud.get_provider_connection(self._db, connection_id) + return resolve_distribution_view( + profile, + release, + distribution_id=distribution_id, + connection=connection, + funding_owner_user_id=self._funding_owner_user_id, + ) + + +def _distribution_errors( + db: Any, protocol: StudyProtocolV1, *, actor_is_admin: bool +) -> list[ValidationError]: + """Resolve every condition's distribution and return typed reasons.""" + return validate_protocol( + protocol, + distribution_resolver=_distribution_resolver(db), + actor_is_admin=actor_is_admin, + ) + + +def _condition_authorization_errors( + db: Any, protocol: StudyProtocolV1, current_user: AuthenticatedUser +) -> None: + """Reject foreign-owner profiles or unauthorized connections (C12). + + A condition may only reference a profile owned by the caller (administrators + may reference any profile), and that profile's connection must be granted to + the profile owner (administrators implicitly hold every connection). + """ + problems: list[dict[str, str]] = [] + for condition in protocol.conditions: + field = f"conditions[{condition.condition_id}].distribution_id" + profile = crud.get_agent_profile_by_id(db, condition.distribution_id) + if profile is None: + problems.append( + { + "code": "AGENT_PROFILE_NOT_FOUND", + "field": field, + "message": "the referenced profile does not exist", + } + ) + continue + if not current_user.is_admin and profile.owner_user_id != current_user.user_id: + problems.append( + { + "code": "PROFILE_FOREIGN_OWNER", + "field": field, + "message": "the referenced profile belongs to another researcher", + } + ) + connection_id = getattr(profile, "connection_id", None) + if connection_id is None: + problems.append( + { + "code": "CONNECTION_MISSING", + "field": field, + "message": "the referenced profile has no provider connection", + } + ) + continue + connection = crud.get_provider_connection(db, connection_id) + if connection is None: + problems.append( + { + "code": "CONNECTION_UNRESOLVED", + "field": field, + "message": "the referenced provider connection does not exist", + } + ) + continue + if not current_user.is_admin and not crud.provider_connection_is_available( + db, connection.connection_id, profile.owner_user_id + ): + problems.append( + { + "code": "CONNECTION_NOT_GRANTED", + "field": field, + "message": "the profile owner is not authorized for this connection", + } + ) + # The selected approval policy must be verified for this exact release. + # Only enforced when real evidence is resolvable (a JSON object); a + # missing/legacy release is already rejected by distribution resolution. + release_id = getattr(profile, "release_id", None) + if release_id: + release_row = registry_store.get_release(db, release_id) + release_json = ( + getattr(release_row, "release_json", None) + if release_row is not None + else None + ) + if isinstance(release_json, dict) and not approval_option_verified( + release_json, profile.approval_policy + ): + problems.append( + { + "code": "APPROVAL_NOT_VERIFIED", + "field": field, + "message": ( + f"approval policy {profile.approval_policy!r} is not " + f"verified for release {release_id!r}" + ), + } + ) + if problems: + raise HTTPException(status_code=422, detail=problems) + + +def _load_study_row(db: Any, study_id: uuid.UUID): + study = store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + return study + + +def _authorize_study( + db: Any, + current_user: AuthenticatedUser, + study_id: uuid.UUID, +) -> Any: + """Authorize an action on one study by ownership (admin bypasses).""" + study = _load_study_row(db, study_id) + require_study_owner(current_user, study) + return study + + +def _ensure_study_access( + app: App, + current_user: AuthenticatedUser, + study_id: uuid.UUID, +) -> None: + """Authorize before the handler opens its own session.""" + if current_user.is_admin: + return + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + finally: + db.close() + + +def _error_payload(errors: list[ValidationError]) -> list[dict[str, Any]]: + return [error.model_dump(mode="json") for error in errors] + + +def _revision_payload(row: Any) -> dict[str, Any]: + return store.revision_summary(row) + + +def _load_lineage(db: Any, study_id: uuid.UUID) -> RevisionLineage: + rows = store.list_revisions(db, study_id) + revisions = [store.row_to_revision(row) for row in rows] + return lineage_from_revisions(study_id, revisions) + + +def _publish( + db: Any, + *, + payload: PublishRequest, + current_user: AuthenticatedUser, +) -> dict[str, Any]: + """Shared publish/supersede body: digest check, then immutable publication. + + Every condition's distribution is resolved and its exact pin is frozen into + the stored revision by :func:`publish_revision`. Foreign-owner profiles and + unauthorized connections are rejected before publication. + """ + actor = current_user.email + actor_is_admin = current_user.is_admin + computed_digest = protocol_digest(payload.protocol) + if ( + payload.expected_protocol_digest is not None + and payload.expected_protocol_digest != computed_digest + ): + raise HTTPException( + status_code=400, + detail=( + "protocol_digest mismatch: document was not built with the " + f"canonical serializer (expected {computed_digest})" + ), + ) + + _condition_authorization_errors(db, payload.protocol, current_user) + + study_row = store.get_study(db, payload.protocol.study_id) + if study_row is None: + raise HTTPException(status_code=404, detail="Study not found") + # The study owner (not the publishing actor) funds the connection grant; it + # is frozen into every condition's agent config at publication. + funding_owner_user_id = study_row.created_by + + resolver = _distribution_resolver( + db, funding_owner_user_id=funding_owner_user_id + ) + errors = validate_protocol( + payload.protocol, + distribution_resolver=resolver, + actor_is_admin=actor_is_admin, + ) + blocking = blocking_errors(errors) + if blocking: + raise HTTPException(status_code=422, detail=_error_payload(errors)) + + lineage = _load_lineage(db, payload.protocol.study_id) + result = publish_revision( + payload.protocol, + lineage, + distribution_resolver=resolver, + actor_is_admin=actor_is_admin, + expected_revision_number=payload.expected_revision_number, + actor=actor, + now=datetime.now(timezone.utc), + ) + + if result.outcome == PublicationOutcome.VALIDATION_FAILED: + raise HTTPException( + status_code=422, detail=_error_payload(blocking_errors(result.errors)) + ) + + advisory = _error_payload(warnings(errors)) + + if result.outcome == PublicationOutcome.CONFLICT: + return { + "outcome": result.outcome.value, + "revision": None, + "conflict": ( + result.conflict.model_dump(mode="json") if result.conflict else None + ), + "errors": [], + "warnings": advisory, + } + + assert result.revision is not None # PUBLISHED always carries a revision + row = store.persist_revision(db, result.revision) + + # Publication reserves the owner's one live-study slot. A partial unique + # index cannot react to time passing, so release any of the owner's studies + # whose end date has passed before reserving the slot; the index then makes + # concurrent publication DB-safe (a loser raises IntegrityError -> 409). + from sqlalchemy.exc import IntegrityError + + if funding_owner_user_id is not None: + store.deactivate_expired_research_studies(db, funding_owner_user_id) + study_row = store.get_study(db, result.revision.study_id) + if study_row is not None and not study_row.is_active: + try: + store.set_study_active(db, result.revision.study_id, True) + except IntegrityError: + db.rollback() + raise HTTPException( + status_code=409, + detail={ + "code": "LIVE_STUDY_CONFLICT", + "message": ( + "this researcher already has a live study; end it before " + "publishing another" + ), + }, + ) + + if result.audit is not None: + store.persist_audit(db, result.audit) + return { + "outcome": result.outcome.value, + "revision": _revision_payload(row), + "conflict": None, + "errors": [], + "warnings": advisory, + } + + +@router.post("/drafts", summary="Create an editable study protocol draft") +def create_draft( + payload: DraftCreateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Persist a draft. Credential/identifier leakage is rejected outright.""" + require_researcher(current_user) + _ensure_study_access(app, current_user, payload.study_id) + + safety_errors = scan_document_safety(payload.protocol) + if safety_errors: + raise HTTPException(status_code=422, detail=_error_payload(safety_errors)) + + db = app.get_db_session() + try: + distribution_errors = _distribution_errors( + db, payload.protocol, actor_is_admin=current_user.is_admin + ) + if blocking_errors(distribution_errors): + raise HTTPException( + status_code=422, detail=_error_payload(distribution_errors) + ) + if store.get_study(db, payload.study_id) is None: + schedule = payload.protocol.schedule + store.create_study( + db, + study_id=payload.study_id, + name=payload.name, + description=payload.protocol.metadata.description, + created_by=current_user.user_id, + starts_at=getattr(schedule, "start_at", None), + ends_at=getattr(schedule, "end_at", None), + is_research=True, + ) + row = store.create_draft( + db, + draft_id=uuid.uuid4(), + study_id=payload.study_id, + name=payload.name, + protocol=payload.protocol, + ) + return JsonResponseWithStatus( + status_code=201, + content={ + "draft_id": str(row.draft_id), + "study_id": str(row.study_id), + "name": row.name, + "schema_version": row.schema_version, + }, + ) + finally: + db.close() + + +@router.get("/drafts", summary="List editable study protocol drafts") +def list_drafts( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + _ensure_study_access(app, current_user, study_id) + db = app.get_db_session() + try: + rows = store.list_drafts(db, study_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "drafts": [ + { + "draft_id": str(row.draft_id), + "study_id": str(row.study_id), + "name": row.name, + "schema_version": row.schema_version, + } + for row in rows + ] + }, + ) + finally: + db.close() + + +@router.post("/drafts/validate", summary="Validate a study protocol document") +def validate_draft( + payload: ValidateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return typed validation reasons; an empty list means publishable.""" + require_researcher(current_user) + db = app.get_db_session() + try: + existing = store.get_study(db, payload.protocol.study_id) + if existing is not None: + require_study_owner(current_user, existing) + errors = _distribution_errors( + db, payload.protocol, actor_is_admin=current_user.is_admin + ) + finally: + db.close() + blocking = blocking_errors(errors) + return JsonResponseWithStatus( + status_code=200 if not blocking else 422, + content={ + "valid": not blocking, + "errors": _error_payload(errors), + "warnings": _error_payload(warnings(errors)), + }, + ) + + +@router.get("/drafts/{draft_id}", summary="Fetch one study protocol draft") +def get_draft( + draft_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + row = store.get_draft(db, draft_id) + if row is None: + raise HTTPException(status_code=404, detail="Study protocol draft not found") + _authorize_study(db, current_user, row.study_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "draft_id": str(row.draft_id), + "study_id": str(row.study_id), + "name": row.name, + "schema_version": row.schema_version, + "protocol": row.protocol_json, + }, + ) + finally: + db.close() + + +@router.post( + "/drafts/{draft_id}/publish", summary="Publish a draft as an immutable revision" +) +def publish_draft( + draft_id: uuid.UUID, + payload: PublishRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Publish a protocol; a stale ``expected_revision_number`` yields a conflict.""" + require_researcher(current_user) + _ensure_study_access(app, current_user, payload.protocol.study_id) + db = app.get_db_session() + try: + if store.get_draft(db, draft_id) is None: + raise HTTPException( + status_code=404, detail="Study protocol draft not found" + ) + content = _publish( + db, + payload=payload, + current_user=current_user, + ) + status_code = ( + 409 + if content["outcome"] == PublicationOutcome.CONFLICT.value + else 201 + ) + return JsonResponseWithStatus(status_code=status_code, content=content) + finally: + db.close() + + +@router.get("/revisions", summary="List a study's immutable revisions") +def list_revisions( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + _ensure_study_access(app, current_user, study_id) + db = app.get_db_session() + try: + rows = store.list_revisions(db, study_id) + return JsonResponseWithStatus( + status_code=200, + content={"revisions": [_revision_payload(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get("/revisions/{revision_id}", summary="Fetch one immutable revision") +def get_revision( + revision_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + row = store.get_revision(db, revision_id) + if row is None: + raise HTTPException(status_code=404, detail="Study revision not found") + _authorize_study(db, current_user, row.study_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "revision": _revision_payload(row), + "protocol": row.protocol_json, + }, + ) + finally: + db.close() + + +@router.post( + "/revisions/{revision_id}/supersede", + summary="Publish a successor revision for an edited protocol", +) +def supersede_revision( + revision_id: uuid.UUID, + payload: PublishRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Create a successor revision; the prior revision is never mutated.""" + require_researcher(current_user) + _ensure_study_access(app, current_user, payload.protocol.study_id) + db = app.get_db_session() + try: + existing = store.get_revision(db, revision_id) + if existing is None: + raise HTTPException(status_code=404, detail="Study revision not found") + content = _publish( + db, + payload=payload, + current_user=current_user, + ) + status_code = ( + 409 + if content["outcome"] == PublicationOutcome.CONFLICT.value + else 201 + ) + return JsonResponseWithStatus(status_code=status_code, content=content) + finally: + db.close() + + +@router.post( + "/revisions/{revision_id}/retire", summary="Retire a published revision" +) +def retire_published_revision( + revision_id: uuid.UUID, + payload: RetireRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Transition a revision to ``RETIRED`` without changing its content. + + Retirement of the last published revision is the terminal "end study" + action: it releases the owner's live-study slot (``is_active = false``). + """ + db = app.get_db_session() + try: + row = store.get_revision(db, revision_id) + if row is None: + raise HTTPException(status_code=404, detail="Study revision not found") + _authorize_study(db, current_user, row.study_id) + result = retire_revision( + store.row_to_revision(row), + actor=payload.actor or current_user.email, + now=datetime.now(timezone.utc), + ) + updated = store.retire_revision(db, revision_id) + # Retiring the last published revision ends the study: it releases the + # owner's single live-study slot and completes/revokes its enrollments. + if result.retired and store.get_latest_published_revision(db, row.study_id) is None: + store.set_study_active(db, row.study_id, False) + identity_store.complete_enrollments_for_study( + db, row.study_id, now=datetime.now(timezone.utc) + ) + if result.audit is not None: + store.persist_audit(db, result.audit) + return JsonResponseWithStatus( + status_code=200, + content={ + "retired": result.retired, + "revision": _revision_payload(updated) if updated is not None else None, + "message": result.message, + }, + ) + finally: + db.close() + + +class EndStudyRequest(BaseModel): + """Optional actor override for an explicit study end.""" + + actor: Optional[str] = Field(default=None) + + +@router.post("/{study_id}/end", summary="End a research study") +def end_study( + study_id: uuid.UUID, + payload: EndStudyRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Explicitly end a study: release the slot and complete its enrollments. + + Owner/admin only. Marks every ACTIVE enrollment ``COMPLETED`` (terminal), bumps + their revocation epoch, revokes their sessions, and frees the owner's + live-study slot so all funded operations and previously issued capabilities + for the study are refused. + """ + now = datetime.now(timezone.utc) + db = app.get_db_session() + try: + study = _authorize_study(db, current_user, study_id) + completed = identity_store.complete_enrollments_for_study( + db, study_id, now=now + ) + store.set_study_active(db, study_id, False) + store.persist_audit( + db, + AuditRecord( + event_type="study.ended", + study_id=study_id, + actor=payload.actor or current_user.email, + occurred_at=now, + detail=f"completed {completed} enrollment(s)", + ), + ) + updated = store.get_study(db, study_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "study": _study_payload(updated) if updated is not None else None, + "completed_enrollments": completed, + }, + ) + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Study index and join code (admin: enumerates the whole deployment) +# --------------------------------------------------------------------------- + + +def _study_index_entry(db: Any, study: Any) -> dict[str, Any]: + """Build one study's index row with its current published revision.""" + latest = store.get_latest_published_revision(db, study.study_id) + return { + "study_id": str(study.study_id), + "name": study.name, + "description": study.description, + "created_by": ( + str(study.created_by) if getattr(study, "created_by", None) else None + ), + "is_research": bool(getattr(study, "is_research", False)), + "is_active": bool(getattr(study, "is_active", False)), + "join_code": getattr(latest, "join_code", None) if latest is not None else None, + "latest_revision": ( + _revision_payload(latest) if latest is not None else None + ), + } + + +def _study_payload(study: Any) -> dict[str, Any]: + created_at = study.created_at + starts_at = getattr(study, "starts_at", None) + ends_at = getattr(study, "ends_at", None) + return { + "study_id": str(study.study_id), + "name": study.name, + "description": study.description, + "owner": study.owner, + "created_by": ( + str(study.created_by) if getattr(study, "created_by", None) else None + ), + "is_research": bool(getattr(study, "is_research", False)), + "is_active": bool(getattr(study, "is_active", False)), + "starts_at": starts_at.isoformat() if isinstance(starts_at, datetime) else starts_at, + "ends_at": ends_at.isoformat() if isinstance(ends_at, datetime) else ends_at, + "created_at": ( + created_at.isoformat() if isinstance(created_at, datetime) else created_at + ), + } + + +@router.post("", summary="Create a study identity") +def create_study( + payload: StudyCreateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Mint a real research Study row owned by the caller. + + An administrator may always create one; an enabled researcher may create + their first study without any per-study role. The row starts inactive, so a + draft does not reserve the owner's one live-study slot. + """ + require_researcher(current_user) + db = app.get_db_session() + try: + study_id = uuid.uuid4() + study = store.create_study( + db, + study_id=study_id, + name=payload.name, + description=payload.description, + created_by=current_user.user_id, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + is_research=True, + ) + return JsonResponseWithStatus( + status_code=201, content={"study": _study_payload(study)} + ) + finally: + db.close() + + +@router.get("", summary="List the caller's studies") +def list_studies( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Owner-scoped study index; administrators see every study.""" + require_researcher(current_user) + db = app.get_db_session() + try: + owner_scope = None if current_user.is_admin else current_user.user_id + studies = store.list_studies(db, owner_scope) + return JsonResponseWithStatus( + status_code=200, + content={"studies": [_study_index_entry(db, study) for study in studies]}, + ) + finally: + db.close() + + +@router.get( + "/{study_id}/join-code", summary="Get a study's current join code and revision" +) +def get_study_join_code( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Owner-scoped: the current join code bound to the live published revision.""" + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + row = store.get_study_join_code(db, study_id) + if row is None: + raise HTTPException( + status_code=404, + detail="Study has no published revision with a join code", + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "study_id": str(study_id), + "join_code": row.join_code, + "revision": _revision_payload(row), + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/telemetry.py b/src/backend/routers/research/telemetry.py new file mode 100644 index 0000000..185209a --- /dev/null +++ b/src/backend/routers/research/telemetry.py @@ -0,0 +1,217 @@ +"""Telemetry ingestion API (Issue 09). + +Mounted under ``/api/research/telemetry``. Batch submission is authenticated +solely by the scoped session capability carried in the request body — a normal +account/session cookie is never accepted in its place. Receipt inspection is +admin/researcher-only. +""" + +from __future__ import annotations + +import logging +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.exc import SQLAlchemyError + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.bootstrap import BOOTSTRAP_SIGNING_SECRET +from research.analysis.operations import store as operations_store +from research.participants import identity as identity_store +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.models import ( + CapabilityVerification, # noqa: TC001 - FastAPI evaluates route annotations at runtime + SessionCapability, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.runtime.sessions import store as session_store +from research.study.protocol import store as protocol_store +from research.telemetry.ingestion.models import ( + TelemetryBatchAckV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + TelemetryBatchRequestV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.telemetry.ingestion.service import ingest_batch +from research.telemetry.ingestion.store import SqlAlchemyIngestionStore, get_receipt_by_id + +router = APIRouter() + +logger = logging.getLogger(__name__) + +AUDIENCE = "research-runtime" +SCOPE_WRITE = "telemetry:write" + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _log_ack(ack: TelemetryBatchAckV1) -> None: + """Emit exactly one compact line per batch so stalled uploads are visible. + + Records only ids, counts and the distinct reason codes — never event + payloads — so a wedged spool (`docker logs backend`) is diagnosable. + """ + rejected_reasons = sorted( + {event.reason.value for event in ack.rejected if event.reason is not None} + ) + retryable_reasons = sorted( + {event.reason.value for event in ack.retryable if event.reason is not None} + ) + logger.warning( + "telemetry batch %s ack: accepted=%d duplicate=%d rejected=%d retryable=%d " + "rejected_reasons=%s retryable_reasons=%s", + ack.batch_id, + len(ack.accepted), + len(ack.duplicate), + len(ack.rejected), + len(ack.retryable), + rejected_reasons or "-", + retryable_reasons or "-", + ) + + +def _verify_session_capability( + capability: SessionCapability, + *, + now: datetime, + current_revocation_epoch: int, + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_revision_id: Optional[uuid.UUID] = None, +) -> CapabilityVerification: + """Verify the batch capability against the subject resolved from its events.""" + return verify_capability( + capability, + BOOTSTRAP_SIGNING_SECRET or "", + expected_audience=AUDIENCE, + expected_scope=[SCOPE_WRITE], + now=now, + current_revocation_epoch=current_revocation_epoch, + expected_enrollment_id=expected_enrollment_id, + expected_research_session_id=expected_research_session_id, + expected_revision_id=expected_revision_id, + ) + + +def _ingestion_kill_switch_check(db, payload: TelemetryBatchRequestV1): + """DB-backed kill-switch predicate scoped to the batch's active subject. + + The batch is anchored by its first in-scope event (matching the ingestion + service's anchor rule), so the check blocks exactly the study/revision/ + enrollment the accepted events would be attributed to. + """ + for event in payload.events: + if ( + event.enrollment_id is None + or event.research_session_id is None + or event.revision_id is None + ): + continue + session_row = session_store.get_session(db, event.research_session_id) + if session_row is None: + continue + enrollment_row = identity_store.get_enrollment(db, event.enrollment_id) + if enrollment_row is None: + continue + revision_row = protocol_store.get_revision(db, session_row.study_revision_id) + study_id = revision_row.study_id if revision_row is not None else None + return operations_store.db_kill_switch_check( + db, + study_id=study_id, + revision_id=session_row.study_revision_id, + enrollment_id=event.enrollment_id, + ) + + def _never_engaged() -> bool: + return False + + return _never_engaged + + +def _enrollment_resolver(db): + def resolve(enrollment_id: uuid.UUID): + # Lock the enrollment row so concurrent batches for the same subject + # serialize inside the ingestion transaction. + row = identity_store.get_enrollment(db, enrollment_id, for_update=True) + return identity_store.row_to_enrollment(row) if row is not None else None + + return resolve + + +def _session_resolver(db): + def resolve(research_session_id: uuid.UUID): + row = session_store.get_session(db, research_session_id, for_update=True) + return session_store.row_to_session(row) if row is not None else None + + return resolve + + +@router.post("/batches", summary="Submit a canonical telemetry batch") +def submit_telemetry_batch( + payload: TelemetryBatchRequestV1, + app: App = Depends(App.get_instance), +): + """Ingest a batch authenticated by its scoped session capability.""" + db = app.get_db_session() + try: + store = SqlAlchemyIngestionStore(db) + ack = ingest_batch( + payload, + capability_verifier=_verify_session_capability, + enrollment_resolver=_enrollment_resolver(db), + session_resolver=_session_resolver(db), + store=store, + now=_now(), + kill_switch_check=_ingestion_kill_switch_check(db, payload), + ) + _log_ack(ack) + return JsonResponseWithStatus( + status_code=200, content=ack.model_dump(mode="json") + ) + except SQLAlchemyError as error: + db.rollback() + raise HTTPException( + status_code=503, detail="telemetry store is unavailable" + ) from error + finally: + db.close() + + +@router.get("/receipts/{receipt_id}", summary="Fetch an immutable batch receipt") +def get_batch_receipt( + receipt_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Admin/researcher view of a stored batch receipt.""" + require_admin(current_user) + db = app.get_db_session() + try: + receipt = get_receipt_by_id(db, receipt_id) + if receipt is None: + raise HTTPException(status_code=404, detail="Batch receipt not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "receipt": receipt.ack.model_dump(mode="json"), + "enrollment_id": ( + str(receipt.enrollment_id) + if receipt.enrollment_id is not None + else None + ), + "research_session_id": ( + str(receipt.research_session_id) + if receipt.research_session_id is not None + else None + ), + }, + ) + finally: + db.close() diff --git a/src/database/crud.py b/src/database/crud.py index 340023c..70aafb3 100644 --- a/src/database/crud.py +++ b/src/database/crud.py @@ -14,10 +14,6 @@ from utils import hash_password, verify_password -class AgentProfileInActiveStudyError(Exception): - pass - - # User def create_user( db: Session, user: Union[Queries.CreateUser, Queries.CreateUserOauth] @@ -234,27 +230,6 @@ def create_behavioral_telemetry( return db_telemetry -def get_contextual_telemetry_by_id( - db: Session, telemetry_id: uuid.UUID -) -> Optional[db_schemas.ContextualTelemetry]: - return ( - db.query(db_schemas.ContextualTelemetry) - .filter(db_schemas.ContextualTelemetry.contextual_telemetry_id == telemetry_id) - .first() - ) - - -def get_behavioral_telemetry_by_id( - db: Session, telemetry_id: uuid.UUID -) -> Optional[db_schemas.BehavioralTelemetry]: - return ( - db.query(db_schemas.BehavioralTelemetry) - .filter(db_schemas.BehavioralTelemetry.behavioral_telemetry_id == telemetry_id) - .first() - ) - - -# MetaQuery operations def create_completion_query( db: Session, query: Queries.CreateCompletionQuery, id: str = "" ) -> db_schemas.CompletionQuery: @@ -341,26 +316,6 @@ def get_meta_query_by_id( ) -def get_completion_query_by_id( - db: Session, meta_query_id: uuid.UUID -) -> Optional[db_schemas.CompletionQuery]: - return ( - db.query(db_schemas.CompletionQuery) - .filter(db_schemas.CompletionQuery.meta_query_id == meta_query_id) - .first() - ) - - -def get_chat_query_by_id( - db: Session, meta_query_id: uuid.UUID -) -> Optional[db_schemas.ChatQuery]: - return ( - db.query(db_schemas.ChatQuery) - .filter(db_schemas.ChatQuery.meta_query_id == meta_query_id) - .first() - ) - - def get_chat_queries_for_chat( db: Session, chat_id: uuid.UUID ) -> list[db_schemas.ChatQuery]: @@ -410,16 +365,6 @@ def create_generation( return db_generation -def get_generations_by_meta_query( - db: Session, meta_query_id: uuid.UUID -) -> list[db_schemas.HadGeneration]: - return ( - db.query(db_schemas.HadGeneration) - .filter(db_schemas.HadGeneration.meta_query_id == meta_query_id) - .all() - ) - - # def update_generation_acceptance( # db: Session, update_data: Queries.UpdateGenerationAcceptance # ) -> Optional[db_schemas.HadGeneration]: @@ -921,16 +866,6 @@ def create_ground_truth( return db_ground_truth -def get_ground_truths_for_completion( - db: Session, completion_query_id: uuid.UUID -) -> list[db_schemas.GroundTruth]: - return ( - db.query(db_schemas.GroundTruth) - .filter(db_schemas.GroundTruth.completion_query_id == completion_query_id) - .all() - ) - - def get_all_programming_languages(db: Session) -> list[db_schemas.ProgrammingLanguage]: return db.query(db_schemas.ProgrammingLanguage).all() @@ -1129,37 +1064,6 @@ def search_similar_documentation( return [] -def regenerate_embeddings(db: Session, language: Optional[str] = None) -> int: - """ - Regenerate embeddings for all documentation entries. - Useful when changing embedding models or fixing corrupted embeddings. - - Args: - language: Optional language filter to regenerate only specific language docs - - Returns: - Number of embeddings regenerated - """ - query = db.query(db_schemas.Documentation) - - if language: - query = query.filter(db_schemas.Documentation.language == language) - - docs = query.all() - - updated_count = 0 - for doc in docs: - try: - new_embedding = encode_text(doc.content) - doc.embedding = new_embedding - updated_count += 1 - except Exception as e: - print(f"Failed to regenerate embedding for doc {doc.documentation_id}: {e}") - - db.commit() - return updated_count - - def get_documentation_stats(db: Session) -> dict: """Get statistics about documentation entries.""" total_docs = db.query(db_schemas.Documentation).count() @@ -1201,39 +1105,35 @@ def get_documentation_stats(db: Session) -> dict: # decision is made. -class DuplicateAgentEventError(ValueError): - """Raised when a self-reported event batch replays already-ingested ids. - - Ingestion is idempotent by ``span_id`` (the runtime's own event id), so a - retried upload is rejected rather than duplicated. - """ - - def __init__(self, event_ids: List[str]) -> None: - self.event_ids = event_ids - super().__init__(f"Duplicate agent event IDs: {', '.join(event_ids)}") - - def create_agent_profile( db: Session, + *, + owner_user_id: uuid.UUID, name: str, model: str, tools_json: str, approval_policy: str, max_steps: int, framework_version: str = "code4me2-agent", - base_url: Optional[str] = None, - api_key_ref: Optional[str] = None, + connection_id: Optional[uuid.UUID] = None, + release_id: Optional[str] = None, is_active: bool = True, max_context_tokens: Optional[int] = None, temperature: Optional[float] = None, ) -> db_schemas.AgentProfile: + """Create a researcher-owned profile template. + + The provider endpoint/secret live on the referenced ``provider_connection``; + a profile never stores a URL or a secret reference. + """ profile = db_schemas.AgentProfile( profile_id=uuid.uuid4(), + owner_user_id=owner_user_id, name=name, model=model, framework_version=framework_version, - base_url=base_url, - api_key_ref=api_key_ref, + connection_id=connection_id, + release_id=release_id, tools_json=tools_json, approval_policy=approval_policy, max_steps=max_steps, @@ -1247,14 +1147,6 @@ def create_agent_profile( return profile -def get_agent_profile(db: Session, name: str) -> Optional[db_schemas.AgentProfile]: - return ( - db.query(db_schemas.AgentProfile) - .filter(db_schemas.AgentProfile.name == name) - .first() - ) - - def get_agent_profile_by_id( db: Session, profile_id: uuid.UUID ) -> Optional[db_schemas.AgentProfile]: @@ -1265,39 +1157,35 @@ def get_agent_profile_by_id( ) -def list_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: - return db.query(db_schemas.AgentProfile).order_by(db_schemas.AgentProfile.name).all() - - -def list_active_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: - """Candidate arms for new A/B assignments — only profiles flagged active. - - Ordered deterministically by name so behaviour is reproducible across calls - (relevant for tests and for any future deterministic bucketing). - """ - return ( - db.query(db_schemas.AgentProfile) - .filter(db_schemas.AgentProfile.is_active.is_(True)) - .order_by(db_schemas.AgentProfile.name) - .all() - ) +def list_agent_profiles( + db: Session, owner_user_id: Optional[uuid.UUID] = None +) -> List[db_schemas.AgentProfile]: + """List profiles; ``owner_user_id`` scopes to one researcher (admin = all).""" + query = db.query(db_schemas.AgentProfile) + if owner_user_id is not None: + query = query.filter(db_schemas.AgentProfile.owner_user_id == owner_user_id) + return query.order_by(db_schemas.AgentProfile.name).all() def update_agent_profile( db: Session, profile_id: uuid.UUID, - name: str, - model: str, - tools_json: str, - approval_policy: str, - max_steps: int, - framework_version: str = "code4me2-agent", - base_url: Optional[str] = None, - api_key_ref: Optional[str] = None, - is_active: bool = True, + *, + name: Optional[str] = None, + model: Optional[str] = None, + tools_json: Optional[str] = None, + approval_policy: Optional[str] = None, + max_steps: Optional[int] = None, + framework_version: Optional[str] = None, + connection_id: Optional[uuid.UUID] = None, + release_id: Optional[str] = None, + is_active: Optional[bool] = None, max_context_tokens: Optional[int] = None, temperature: Optional[float] = None, + update_connection_id: bool = False, + update_release_id: bool = False, ) -> Optional[db_schemas.AgentProfile]: + """Update a profile template in place (caller performs ownership checks).""" profile = ( db.query(db_schemas.AgentProfile) .filter(db_schemas.AgentProfile.profile_id == profile_id) @@ -1306,25 +1194,35 @@ def update_agent_profile( ) if profile is None: return None - if is_agent_profile_in_active_study(db, profile_id): - raise AgentProfileInActiveStudyError - profile.name = name - profile.model = model - profile.framework_version = framework_version - profile.base_url = base_url - profile.api_key_ref = api_key_ref - profile.tools_json = tools_json - profile.approval_policy = approval_policy - profile.max_steps = max_steps - profile.is_active = is_active - profile.max_context_tokens = max_context_tokens - profile.temperature = temperature + if name is not None: + profile.name = name + if model is not None: + profile.model = model + if framework_version is not None: + profile.framework_version = framework_version + if tools_json is not None: + profile.tools_json = tools_json + if approval_policy is not None: + profile.approval_policy = approval_policy + if max_steps is not None: + profile.max_steps = max_steps + if update_connection_id: + profile.connection_id = connection_id + if update_release_id: + profile.release_id = release_id + if is_active is not None: + profile.is_active = is_active + if max_context_tokens is not None: + profile.max_context_tokens = max_context_tokens + if temperature is not None: + profile.temperature = temperature db.commit() db.refresh(profile) return profile def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: + """Archive a profile template (never erase a frozen study snapshot).""" profile = ( db.query(db_schemas.AgentProfile) .filter(db_schemas.AgentProfile.profile_id == profile_id) @@ -1333,8 +1231,6 @@ def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: ) if profile is None: return False - if is_agent_profile_in_active_study(db, profile_id): - raise AgentProfileInActiveStudyError if not profile.is_active: return True profile.is_active = False @@ -1342,261 +1238,141 @@ def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: return True -# ── Agent profile assignments (A/B testing) ───────────────────────────────── +# ── Provider connections and grants ───────────────────────────────────────── -def get_agent_profile_assignment( - db: Session, user_id: uuid.UUID -) -> Optional[db_schemas.AgentProfileAssignment]: - return ( - db.query(db_schemas.AgentProfileAssignment) - .filter(db_schemas.AgentProfileAssignment.user_id == user_id) - .first() +def create_provider_connection( + db: Session, + *, + label: str, + base_url: str, + secret_ref: str, + models_json: str, + is_active: bool = True, +) -> db_schemas.ProviderConnection: + connection = db_schemas.ProviderConnection( + connection_id=uuid.uuid4(), + label=label, + base_url=base_url, + secret_ref=secret_ref, + models_json=models_json, + is_active=is_active, ) + db.add(connection) + db.commit() + db.refresh(connection) + return connection -def get_agent_study_assignment( - db: Session, study_id: uuid.UUID, user_id: uuid.UUID -) -> Optional[db_schemas.AgentStudyAssignment]: +def get_provider_connection( + db: Session, connection_id: uuid.UUID +) -> Optional[db_schemas.ProviderConnection]: + return db.get(db_schemas.ProviderConnection, connection_id) + + +def get_provider_connection_by_label( + db: Session, label: str +) -> Optional[db_schemas.ProviderConnection]: return ( - db.query(db_schemas.AgentStudyAssignment) - .filter( - db_schemas.AgentStudyAssignment.study_id == study_id, - db_schemas.AgentStudyAssignment.user_id == user_id, - ) + db.query(db_schemas.ProviderConnection) + .filter(db_schemas.ProviderConnection.label == label) .first() ) -def create_agent_study_assignment( - db: Session, - *, - study_id: uuid.UUID, - user_id: uuid.UUID, - profile_id: uuid.UUID, - arm_name: str, - is_baseline: bool, - source: str, -) -> Optional[db_schemas.AgentStudyAssignment]: - assignment_id = uuid.uuid4() - inserted_assignment_id = db.execute( - pg_insert(db_schemas.AgentStudyAssignment) - .values( - assignment_id=assignment_id, - study_id=study_id, - user_id=user_id, - profile_id=profile_id, - arm_name=arm_name, - is_baseline=is_baseline, - source=source, - ) - .on_conflict_do_nothing(index_elements=["study_id", "user_id"]) - .returning(db_schemas.AgentStudyAssignment.assignment_id) - ).scalar_one_or_none() - if inserted_assignment_id is None: - return None - db.commit() - return db.get(db_schemas.AgentStudyAssignment, inserted_assignment_id) - - -def list_agent_study_assignments( - db: Session, -) -> List[db_schemas.AgentStudyAssignment]: +def list_provider_connections(db: Session) -> List[db_schemas.ProviderConnection]: return ( - db.query(db_schemas.AgentStudyAssignment) - .order_by(db_schemas.AgentStudyAssignment.assigned_at.desc()) + db.query(db_schemas.ProviderConnection) + .order_by(db_schemas.ProviderConnection.label) .all() ) -def delete_agent_study_assignments( - db: Session, user_id: uuid.UUID, study_id: Optional[uuid.UUID] = None -) -> int: - query = db.query(db_schemas.AgentStudyAssignment).filter( - db_schemas.AgentStudyAssignment.user_id == user_id - ) - if study_id is not None: - query = query.filter(db_schemas.AgentStudyAssignment.study_id == study_id) - deleted = query.delete() - db.commit() - return deleted - - -def set_agent_profile_assignment( +def update_provider_connection( db: Session, - user_id: uuid.UUID, - profile_id: uuid.UUID, - source: str = "auto", -) -> db_schemas.AgentProfileAssignment: - """Upsert a user's profile assignment. - - Used both for the first-contact random draw (source="auto") and for admin - overrides (source="manual"). Re-assigning overwrites the existing row. - """ - assignment = get_agent_profile_assignment(db, user_id) - if assignment is None: - assignment = db_schemas.AgentProfileAssignment( - user_id=user_id, - profile_id=profile_id, - source=source, - ) - db.add(assignment) - else: - assignment.profile_id = profile_id - assignment.source = source - assignment.assigned_at = datetime.now() + connection_id: uuid.UUID, + *, + label: Optional[str] = None, + base_url: Optional[str] = None, + secret_ref: Optional[str] = None, + models_json: Optional[str] = None, + is_active: Optional[bool] = None, +) -> Optional[db_schemas.ProviderConnection]: + connection = db.get(db_schemas.ProviderConnection, connection_id) + if connection is None: + return None + if label is not None: + connection.label = label + if base_url is not None: + connection.base_url = base_url + if secret_ref is not None: + connection.secret_ref = secret_ref + if models_json is not None: + connection.models_json = models_json + if is_active is not None: + connection.is_active = is_active db.commit() - db.refresh(assignment) - return assignment + db.refresh(connection) + return connection -def delete_agent_profile_assignment(db: Session, user_id: uuid.UUID) -> bool: - result = ( - db.query(db_schemas.AgentProfileAssignment) - .filter(db_schemas.AgentProfileAssignment.user_id == user_id) - .delete() - ) +def delete_provider_connection(db: Session, connection_id: uuid.UUID) -> bool: + connection = db.get(db_schemas.ProviderConnection, connection_id) + if connection is None: + return False + db.delete(connection) db.commit() - return result > 0 - - -def list_agent_profile_assignments( - db: Session, -) -> List[db_schemas.AgentProfileAssignment]: - return db.query(db_schemas.AgentProfileAssignment).all() - - -# ── Study ↔ agent profile links (A/B studies for agents) ──────────────────── - - -def set_study_agent_profiles( - db: Session, - study_id: uuid.UUID, - profile_ids: List[uuid.UUID], - baseline_profile_id: Optional[uuid.UUID] = None, -) -> None: - """Replace the agent-profile arms attached to a study. - - Idempotent: clears any existing rows for the study and writes one row per - profile, flagging ``baseline_profile_id`` (if given) as the baseline arm. - Does not commit — the caller owns the transaction (study create/activate - writes several tables atomically). - """ - db.query(db_schemas.StudyAgentProfile).filter( - db_schemas.StudyAgentProfile.study_id == study_id - ).delete() - for profile_id in profile_ids: - db.add( - db_schemas.StudyAgentProfile( - study_id=study_id, - profile_id=profile_id, - is_baseline=(profile_id == baseline_profile_id), - ) - ) + return True -def list_study_agent_profiles( - db: Session, study_id: uuid.UUID -) -> List[db_schemas.StudyAgentProfile]: - """All agent-profile arm links for a study (with the joined profile loaded).""" +def list_available_provider_connections( + db: Session, user_id: uuid.UUID +) -> List[db_schemas.ProviderConnection]: + """Active admin-managed connections available to an authorized researcher.""" return ( - db.query(db_schemas.StudyAgentProfile) - .filter(db_schemas.StudyAgentProfile.study_id == study_id) + db.query(db_schemas.ProviderConnection) + .filter(db_schemas.ProviderConnection.is_active.is_(True)) + .order_by(db_schemas.ProviderConnection.label) .all() ) -def get_study_agent_profile( - db: Session, study_id: uuid.UUID, profile_id: uuid.UUID -) -> Optional[db_schemas.StudyAgentProfile]: +def provider_connection_is_available( + db: Session, connection_id: uuid.UUID, user_id: uuid.UUID +) -> bool: + """Whether an active admin-managed connection can be selected.""" return ( - db.query(db_schemas.StudyAgentProfile) + db.query(db_schemas.ProviderConnection) .filter( - db_schemas.StudyAgentProfile.study_id == study_id, - db_schemas.StudyAgentProfile.profile_id == profile_id, - ) - .first() - ) - - -def get_active_agent_study(db: Session) -> Optional[db_schemas.Study]: - """Return the active study when it has at least one active agent arm.""" - return ( - db.query(db_schemas.Study) - .join( - db_schemas.StudyAgentProfile, - db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, - ) - .join( - db_schemas.AgentProfile, - db_schemas.AgentProfile.profile_id - == db_schemas.StudyAgentProfile.profile_id, + db_schemas.ProviderConnection.connection_id == connection_id, + db_schemas.ProviderConnection.is_active.is_(True), ) - .filter(db_schemas.Study.is_active.is_(True)) - .filter(db_schemas.AgentProfile.is_active.is_(True)) - .first() - ) - - -def is_agent_profile_in_active_study(db: Session, profile_id: uuid.UUID) -> bool: - return ( - db.query(db_schemas.StudyAgentProfile) - .join( - db_schemas.Study, - db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, - ) - .filter(db_schemas.StudyAgentProfile.profile_id == profile_id) - .filter(db_schemas.Study.is_active.is_(True)) .first() is not None ) -def list_active_study_agent_profiles(db: Session) -> List[db_schemas.AgentProfile]: - """Candidate arms drawn from the currently-active study's selected profiles. +# ── Researcher enablement ──────────────────────────────────────────────────── - Returns only profiles that are both attached to the active study and still - flagged ``is_active``. Empty when there is no active study, or the active - study selected no agent profiles — callers fall back to - ``list_active_agent_profiles`` in that case (backwards compatible). - Ordered by name for reproducibility, matching ``list_active_agent_profiles``. - """ - return ( - db.query(db_schemas.AgentProfile) - .join( - db_schemas.StudyAgentProfile, - db_schemas.StudyAgentProfile.profile_id - == db_schemas.AgentProfile.profile_id, - ) - .join( - db_schemas.Study, - db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, - ) - .filter(db_schemas.Study.is_active.is_(True)) - .filter(db_schemas.AgentProfile.is_active.is_(True)) - .order_by(db_schemas.AgentProfile.name) - .all() - ) +def set_user_can_research( + db: Session, user_id: uuid.UUID, enabled: bool +) -> Optional[db_schemas.User]: + """Administrator toggle of the single researcher-enablement flag.""" + user = db.get(db_schemas.User, user_id) + if user is None: + return None + user.can_research = enabled + db.commit() + db.refresh(user) + return user -def list_active_study_agent_profile_links( - db: Session, -) -> List[db_schemas.StudyAgentProfile]: - """Return the arm links of the active agent study, if one exists.""" + +def list_researchers(db: Session) -> List[db_schemas.User]: return ( - db.query(db_schemas.StudyAgentProfile) - .join( - db_schemas.Study, - db_schemas.Study.study_id == db_schemas.StudyAgentProfile.study_id, - ) - .join( - db_schemas.AgentProfile, - db_schemas.AgentProfile.profile_id - == db_schemas.StudyAgentProfile.profile_id, - ) - .filter(db_schemas.Study.is_active.is_(True)) - .filter(db_schemas.AgentProfile.is_active.is_(True)) - .order_by(db_schemas.AgentProfile.name) + db.query(db_schemas.User) + .filter(db_schemas.User.can_research.is_(True)) + .order_by(db_schemas.User.name) .all() ) @@ -1617,6 +1393,7 @@ def create_agent_task( framework_version: Optional[str] = None, source: str = "plugin", owner_user_id: Optional[uuid.UUID] = None, + funding_owner_user_id: Optional[uuid.UUID] = None, owner_project_id: Optional[uuid.UUID] = None, external_run_id: Optional[str] = None, agent_session_id: Optional[str] = None, @@ -1629,6 +1406,9 @@ def create_agent_task( study_arm_name: Optional[str] = None, study_arm_is_baseline: Optional[bool] = None, consent_content_storage: Optional[bool] = None, + research_session_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, + study_revision_id: Optional[uuid.UUID] = None, ) -> db_schemas.AgentTask: task = db_schemas.AgentTask( task_id=task_id or uuid.uuid4(), @@ -1642,6 +1422,7 @@ def create_agent_task( source=source, session_id=session_id, owner_user_id=owner_user_id, + funding_owner_user_id=funding_owner_user_id, owner_project_id=owner_project_id, external_run_id=external_run_id, agent_session_id=agent_session_id, @@ -1651,6 +1432,9 @@ def create_agent_task( study_arm_name=study_arm_name, study_arm_is_baseline=study_arm_is_baseline, consent_content_storage=consent_content_storage, + research_session_id=research_session_id, + enrollment_id=enrollment_id, + study_revision_id=study_revision_id, started_at=started_at, policy_snapshot=policy_snapshot, task_description=task_description, @@ -1792,16 +1576,6 @@ def get_last_agent_event_for_session( return q.order_by(db_schemas.AgentEvent.created_at.desc()).first() -def count_agent_events_for_task(db: Session, task_id: uuid.UUID) -> int: - """Return the number of persisted events for a task.""" - return ( - db.query(func.count(db_schemas.AgentEvent.event_id)) - .filter(db_schemas.AgentEvent.task_id == task_id) - .scalar() - or 0 - ) - - def reserve_agent_event_indexes( db: Session, task_id: uuid.UUID, event_count: int ) -> int: @@ -1991,58 +1765,6 @@ def append_agent_event( # ── Agent edits (human-in-the-loop decisions) ─────────────────────────────── -def create_agent_edit( - db: Session, - task_id: uuid.UUID, - file_path: str, - diff_text: Optional[str] = None, -) -> db_schemas.AgentEdit: - edit = db_schemas.AgentEdit( - edit_id=uuid.uuid4(), - task_id=task_id, - file_path=file_path, - diff_text=diff_text, - ) - db.add(edit) - db.commit() - db.refresh(edit) - return edit - - -def get_agent_edits_by_task( - db: Session, task_id: uuid.UUID -) -> List[db_schemas.AgentEdit]: - return ( - db.query(db_schemas.AgentEdit) - .filter(db_schemas.AgentEdit.task_id == task_id) - .all() - ) - - -def update_agent_edit_decision( - db: Session, - edit_id: uuid.UUID, - was_accepted: bool, - was_modified: Optional[bool] = None, - edit_delta_json: Optional[str] = None, -) -> bool: - data: dict = {"was_accepted": was_accepted, "decided_at": datetime.now()} - if was_modified is not None: - data["was_modified"] = was_modified - if edit_delta_json is not None: - data["edit_delta_json"] = edit_delta_json - result = ( - db.query(db_schemas.AgentEdit) - .filter(db_schemas.AgentEdit.edit_id == edit_id) - .update(data) - ) - db.commit() - return result > 0 - - -# ── Agent memory (survives IDE restarts) ──────────────────────────────────── - - def get_agent_memory_by_session_id( db: Session, session_id: str ) -> Optional[db_schemas.AgentMemory]: diff --git a/src/database/db_schemas.py b/src/database/db_schemas.py index ed40d98..fcbc9f5 100644 --- a/src/database/db_schemas.py +++ b/src/database/db_schemas.py @@ -14,6 +14,7 @@ """ from datetime import datetime +from enum import Enum from pgvector.sqlalchemy import Vector from sqlalchemy import ( @@ -31,6 +32,8 @@ String, Text, UniqueConstraint, + func, + text, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import relationship @@ -63,6 +66,14 @@ STORE_AGENT_CONTENT_DEFAULT = True +class ResearchStudyStatus(str, Enum): + """Lifecycle states for research studies on the shared ``study`` table.""" + + DRAFT = "DRAFT" + ACTIVE = "ACTIVE" + STUDY_STOPPED = "STUDY_STOPPED" + + class Config(Base): """ Configuration settings storage table. @@ -97,6 +108,8 @@ class User(Base): Index("idx_user_config_id", "config_id"), # Index on admin status for efficient filtering Index("idx_user_is_admin", "is_admin"), + # Index on researcher enablement for admin/ownership lookups. + Index("idx_user_can_research", "can_research"), {"schema": "public"}, ) @@ -123,6 +136,13 @@ class User(Base): is_admin = Column( Boolean, server_default="false", default=False ) # Admin status for access control + # Researcher enablement (P3): an administrator flips this on to let an + # account own private profiles and studies. It is the *only* researcher + # authority — there is no per-study role table. Never settable by the + # account itself (no self-promotion route). + can_research = Column( + Boolean, server_default="false", default=False, nullable=False + ) # Relationship to configuration data config = relationship("Config") @@ -470,6 +490,8 @@ class MetaQuery(Base): Index("idx_meta_query_project_id", "project_id"), Index("idx_meta_query_session_id", "session_id"), Index("idx_meta_query_type", "query_type"), + Index("idx_meta_query_timestamp", "timestamp"), + Index("idx_meta_query_timestamp_type", "timestamp", "query_type"), {"schema": "public"}, ) @@ -575,6 +597,9 @@ class HadGeneration(Base): PrimaryKeyConstraint("meta_query_id", "model_id"), # Index for efficient query-model lookups Index("idx_had_generation_meta_query_model", "meta_query_id", "model_id"), + Index("idx_had_generation_confidence", "confidence"), + Index("idx_had_generation_model_id", "model_id"), + Index("idx_had_generation_was_accepted", "was_accepted"), {"schema": "public"}, ) @@ -649,36 +674,79 @@ class Documentation(Base): embedding = Column( Vector(384), nullable=True ) # 384 dimensions for all-MiniLM-L6-v2 - created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + created_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) class Study(Base): """ - A/B testing and user experiment management. - - Manages user studies for configuration testing and analysis. - Only one study can be active at a time per server instance. + Shared study identity for both completion A/B studies and research studies. + + Completion studies (``is_research == False``) keep the completion-specific + required ``default_config_id`` — validated in code, not by a NOT NULL column, + because a research study must not fabricate a completion config. Research + studies set ``is_research`` True and are owner-scoped; their lifecycle and + fixed configuration live in the research-only columns below. At most one + live (active) research study per owner is enforced by the partial unique index + ``uq_study_owner_live_research``. Legacy completion rows leave those research + columns nullable and continue using the existing completion fields. """ - + __tablename__ = "study" __table_args__ = ( Index("idx_study_is_active", "is_active"), Index("idx_study_created_by", "created_by"), Index("idx_study_starts_at", "starts_at"), Index("idx_study_ends_at", "ends_at"), + Index("idx_study_research_status", "research_status"), + # One live research study per owner. Drafts never reserve the slot. + Index( + "uq_study_owner_live_research", + "created_by", + unique=True, + postgresql_where=text("is_research AND is_active"), + ), + Index( + "uq_study_research_join_code", + "join_code", + unique=True, + postgresql_where=text("is_research AND join_code IS NOT NULL"), + ), {"schema": "public"}, ) - + study_id = Column(UUID(as_uuid=True), primary_key=True) name = Column(Text, nullable=False) description = Column(Text) - created_by = Column(UUID(as_uuid=True), ForeignKey("public.user.user_id"), nullable=False) + created_by = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + ) starts_at = Column(DateTime(timezone=True), nullable=False) ends_at = Column(DateTime(timezone=True), nullable=True) is_active = Column(Boolean, server_default="false", default=False) - default_config_id = Column(BigInteger, ForeignKey("public.config.config_id"), nullable=False) - created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) - + # Nullable at the DB level: required for completion studies, absent for + # agent research studies (enforced by request validation). + default_config_id = Column( + BigInteger, ForeignKey("public.config.config_id"), nullable=True + ) + # True for researcher-authored studies; False for legacy completion studies. + is_research = Column(Boolean, server_default="false", default=False, nullable=False) + # Research-only lifecycle/configuration authority. Legacy completion rows + # leave these nullable and continue using their existing fields. + research_status = Column(String, nullable=True) + research_config_json = Column(JSONB, nullable=True) + research_config_digest = Column(String, nullable=True) + join_code = Column(String, nullable=True) + consent_locked_at = Column(DateTime(timezone=True), nullable=True) + stopped_at = Column(DateTime(timezone=True), nullable=True) + stopped_by = Column(String, nullable=True) + created_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) + # Relationships creator = relationship("User") default_config = relationship("Config") @@ -700,10 +768,20 @@ class ConfigAssignmentHistory(Base): {"schema": "public"}, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("public.user.user_id"), primary_key=True) - study_id = Column(UUID(as_uuid=True), ForeignKey("public.study.study_id"), primary_key=True) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + primary_key=True, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + primary_key=True, + ) assigned_config_id = Column(BigInteger, ForeignKey("public.config.config_id"), nullable=False) - assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + assigned_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) # Relationships user = relationship("User") @@ -733,40 +811,82 @@ class ConfigAssignmentHistory(Base): # server-side, never taken from a client-supplied flag. +class ProviderConnection(Base): + """An administrator-maintained upstream provider endpoint. + + A connection owns the provider endpoint (``base_url``) and the *name* of the + deployment secret that holds its key (``secret_ref`` — an environment + variable name, never the value). ``models_json`` is the JSON array of model + names the connection is allowed to serve. Researchers never see + ``base_url``/``secret_ref``; they select a granted connection by id and a + model from ``models_json``. The secret value is resolved from the + environment only at inference time and never persisted or returned. + """ + + __tablename__ = "provider_connection" + __table_args__ = ( + Index("idx_provider_connection_is_active", "is_active"), + {"schema": "public"}, + ) + + connection_id = Column(UUID(as_uuid=True), primary_key=True) + label = Column(String, nullable=False, unique=True) + base_url = Column(String, nullable=False) + # Name of the environment variable holding the upstream key — never the key. + secret_ref = Column(String, nullable=False) + # JSON array of allowed model names for this connection. + models_json = Column(Text, nullable=False) + is_active = Column(Boolean, server_default="true", default=True, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + class AgentProfile(Base): - """Named experiment variant — defines runtime, provider, tools and policy. - - ``framework_version`` selects which agent runtime the profile targets - (``code4me2-agent``, ``goose``, ``codex``), and therefore which telemetry - path a task created from it will use. - - The provider triple (``base_url``, ``api_key_ref``, ``model``) is - deliberately generic: any endpoint speaking the OpenAI-compatible - chat-completions wire format works, so a local Ollama install, Groq, - OpenRouter and OpenAI itself are all a config change rather than a code - change. ``base_url`` NULL falls back to the server's configured default - (see ``agents.provider.resolve_upstream``). + """A researcher-owned, editable agent configuration template. + + A profile fixes the runtime, the provider connection/model, the tool + allowlist, the release artifact pin and the approval policy. It is a + *template*: publishing a study freezes a copy into the study revision, so + later edits never change historical display or an already-published study's + execution. ``connection_id`` names an administrator-managed + :class:`ProviderConnection`; the provider endpoint and secret live there, + never on the profile. """ __tablename__ = "agent_profile" __table_args__ = ( Index("idx_agent_profile_is_active", "is_active"), + Index("idx_agent_profile_owner_user_id", "owner_user_id"), + # Profile names are unique per researcher, never globally. + UniqueConstraint("owner_user_id", "name", name="uq_agent_profile_owner_name"), {"schema": "public"}, ) profile_id = Column(UUID(as_uuid=True), primary_key=True) - name = Column(String, unique=True, nullable=False) + # The researcher who owns this private profile template. + owner_user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + ) + name = Column(String, nullable=False) model = Column(String, nullable=False) # Agent runtime this profile targets: code4me2-agent | goose | codex framework_version = Column( String, nullable=False, server_default="code4me2-agent" ) - # OpenAI-compatible base URL, e.g. http://localhost:11434/v1 (Ollama), - # https://api.groq.com/openai/v1, https://api.openai.com/v1. NULL = server default. - base_url = Column(String, nullable=True) - # Name of the environment variable holding the upstream API key — never the - # key itself, so profiles stay safe to dump from the admin UI or the DB. - api_key_ref = Column(String, nullable=True) + # The exact approved artifact this profile runs (agent_release.release_id). + # The release row owns artifact identity; the profile only pins it. + release_id = Column( + String, + ForeignKey("public.agent_release.release_id", ondelete="RESTRICT"), + nullable=True, + ) + # Administrator-managed provider endpoint + secret owner. + connection_id = Column( + UUID(as_uuid=True), + ForeignKey("public.provider_connection.connection_id", ondelete="RESTRICT"), + nullable=True, + ) tools_json = Column(Text, nullable=False) # JSON array of tool names approval_policy = Column(String, nullable=False) # suggestion_only | per_step | … max_steps = Column(Integer, nullable=False) @@ -776,82 +896,13 @@ class AgentProfile(Base): max_context_tokens = Column( Integer, nullable=True ) # per-turn rolling window; NULL = model max - # Only active profiles are candidate arms for new A/B assignments. Inactive - # profiles stay in the table (drafts, or arms retired mid-study) and existing - # users keep any assignment already pinned to them. + # Only active profiles are candidate arms for new assignments. Inactive + # profiles stay in the table (drafts, or arms retired mid-study). is_active = Column(Boolean, server_default="true", default=True, nullable=False) created_at = Column(DateTime, default=datetime.now) - -class AgentProfileAssignment(Base): - """Sticky user→profile mapping for A/B testing. - - Resolution is server-authoritative: on a user's first agent task we draw a - random active profile and persist it here, so the user keeps the same arm for - the life of the experiment regardless of later changes to the active set. - `source` distinguishes randomized assignments ("auto") from admin overrides - ("manual"); manual rows are excluded from re-rolls and from A/B analysis. - """ - - __tablename__ = "agent_profile_assignment" - __table_args__ = ( - Index("idx_agent_profile_assignment_profile_id", "profile_id"), - {"schema": "public"}, - ) - - user_id = Column( - UUID(as_uuid=True), - ForeignKey("public.user.user_id", ondelete="CASCADE"), - primary_key=True, - ) - profile_id = Column( - UUID(as_uuid=True), - ForeignKey("public.agent_profile.profile_id", ondelete="CASCADE"), - nullable=False, - ) - source = Column(String, nullable=False, server_default="auto") # auto | manual - assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) - - # Relationships - user = relationship("User") - profile = relationship("AgentProfile") - - -class AgentStudyAssignment(Base): - """User-to-arm assignment within one specific study.""" - - __tablename__ = "agent_study_assignment" - __table_args__ = ( - Index("idx_agent_study_assignment_study_id", "study_id"), - Index("idx_agent_study_assignment_profile_id", "profile_id"), - UniqueConstraint("study_id", "user_id", name="uq_agent_study_assignment"), - {"schema": "public"}, - ) - - assignment_id = Column(UUID(as_uuid=True), primary_key=True) - study_id = Column( - UUID(as_uuid=True), - ForeignKey("public.study.study_id", ondelete="RESTRICT"), - nullable=False, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("public.user.user_id", ondelete="RESTRICT"), - nullable=False, - ) - profile_id = Column( - UUID(as_uuid=True), - ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), - nullable=False, - ) - arm_name = Column(String, nullable=False) - is_baseline = Column(Boolean, server_default="false", default=False, nullable=False) - source = Column(String, nullable=False, server_default="auto") - assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) - - study = relationship("Study") - user = relationship("User") - profile = relationship("AgentProfile") + owner = relationship("User") + connection = relationship("ProviderConnection") class AgentTask(Base): @@ -873,6 +924,8 @@ class AgentTask(Base): Index("idx_agent_task_study_id", "study_id"), Index("idx_agent_task_study_assignment_id", "study_assignment_id"), Index("idx_agent_task_profile_id", "profile_id"), + Index("idx_agent_task_research_session_id", "research_session_id"), + Index("idx_agent_task_enrollment_id", "enrollment_id"), {"schema": "public"}, ) @@ -889,6 +942,13 @@ class AgentTask(Base): owner_user_id = Column( UUID(as_uuid=True), ForeignKey("public.user.user_id"), nullable=True ) + # The researcher whose provider connection funds this task's inference. + # Distinct from owner_user_id, which is the participant who ran the task. + funding_owner_user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="SET NULL"), + nullable=True, + ) owner_project_id = Column(UUID(as_uuid=True), nullable=True) # The runtime's *own* identifiers, which are not guaranteed to be UUIDs # (code4me2-agent uses uuid4().hex; ACP session ids are opaque strings). @@ -899,16 +959,19 @@ class AgentTask(Base): study_id = Column(UUID(as_uuid=True), ForeignKey("public.study.study_id"), nullable=True) study_assignment_id = Column( UUID(as_uuid=True), - ForeignKey("public.agent_study_assignment.assignment_id", ondelete="SET NULL"), + ForeignKey("public.study_assignment.assignment_id", ondelete="SET NULL"), nullable=True, ) + # Explicit phase-05 research attribution. Resolved server-side from the + # authorized account + frozen assignment; NULL for ordinary non-study use + # (an explicit "no research context" state, never a guessed one). + research_session_id = Column(UUID(as_uuid=True), nullable=True) + enrollment_id = Column(UUID(as_uuid=True), nullable=True) profile_id = Column( UUID(as_uuid=True), ForeignKey("public.agent_profile.profile_id"), nullable=True, ) - study_arm_name = Column(String, nullable=True) - study_arm_is_baseline = Column(Boolean, nullable=True) agent_profile = Column(String, nullable=False) model = Column(String, nullable=False) # Snapshotted from the assigned profile at task-creation time, like `model`, so @@ -954,7 +1017,6 @@ class AgentTask(Base): task_description = Column(Text, nullable=True) study = relationship("Study") - study_assignment = relationship("AgentStudyAssignment") profile = relationship("AgentProfile") @@ -1124,41 +1186,6 @@ class AgentMemory(Base): updated_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) -class StudyAgentProfile(Base): - """Links a study to the agent profiles that serve as its experiment arms. - - A study with rows here drives agent-profile A/B assignment: its selected - profiles become the candidate pool for new auto-draws (see - ``registry.resolve_assignment``). Studies with no rows here are - completion-only and behave exactly as before this table existed. - - ``is_baseline`` marks the arm that agent-evaluation uplift is measured - against (analogous to ``study.default_config_id`` on the completion side). - """ - - __tablename__ = "study_agent_profile" - __table_args__ = ( - Index("idx_study_agent_profile_profile_id", "profile_id"), - {"schema": "public"}, - ) - - study_id = Column( - UUID(as_uuid=True), - ForeignKey("public.study.study_id", ondelete="CASCADE"), - primary_key=True, - ) - profile_id = Column( - UUID(as_uuid=True), - ForeignKey("public.agent_profile.profile_id"), - primary_key=True, - ) - is_baseline = Column(Boolean, server_default="false", default=False, nullable=False) - - # Relationships - study = relationship("Study") - profile = relationship("AgentProfile") - - # # class SessionQuery(Base): # __tablename__ = "session_queries" @@ -1186,3 +1213,11 @@ class StudyAgentProfile(Base): # UniqueConstraint("session_id", "query_id", name="unique_session_query"), # Index("idx_session_queries_query_id", "query_id"), # ) + + +# ``AgentProfile.release_id`` references the research ``agent_release`` table by +# string name. Importing the research namespace here registers its tables on the +# shared ``Base`` metadata, so ordering/creating the operational metadata works +# regardless of which namespace an entrypoint imports first. ``research_schemas`` +# never imports ``db_schemas``, so this is not an import cycle. +from . import research_schemas as _research_schemas # noqa: E402,F401 (registers research tables on Base) diff --git a/src/database/migration/env.py b/src/database/migration/env.py index 933d52a..064cc8c 100644 --- a/src/database/migration/env.py +++ b/src/database/migration/env.py @@ -19,8 +19,12 @@ src_dir = os.path.join(project_root, "src") sys.path.insert(0, src_dir) -# Import database models +# Import every model namespace that declares tables. ``db_schemas`` owns the +# operational (Code4Me) tables and ``research_schemas`` owns the research tables; +# both attach to the same ``Base`` from ``database.db``, so importing both is +# what makes autogenerate/``alembic check`` see the research tables at all. try: + from database import research_schemas as _research_schemas # noqa: F401 from database.db_schemas import Base target_metadata = Base.metadata @@ -67,6 +71,7 @@ def run_migrations_offline() -> None: dialect_opts={"paramstyle": "named"}, compare_type=True, compare_server_default=True, + include_schemas=True, ) with context.begin_transaction(): @@ -82,6 +87,7 @@ def run_migrations_online() -> None: configuration, prefix="sqlalchemy.", poolclass=pool.NullPool, + connect_args={"options": "-csearch_path=code4me_migration"}, ) with connectable.connect() as connection: @@ -90,6 +96,8 @@ def run_migrations_online() -> None: target_metadata=target_metadata, compare_type=True, compare_server_default=True, + include_schemas=True, + version_table_schema="public", ) with context.begin_transaction(): diff --git a/src/database/migration/versions/8a0084080b46_consolidated_schema.py b/src/database/migration/versions/8a0084080b46_consolidated_schema.py new file mode 100644 index 0000000..0dcf83a --- /dev/null +++ b/src/database/migration/versions/8a0084080b46_consolidated_schema.py @@ -0,0 +1,554 @@ +"""consolidated schema + +Revision ID: 8a0084080b46 +Revises: +Create Date: 2026-09-18 07:43:13.099261 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '8a0084080b46' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade database schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('acp_capability_receipt', + sa.Column('receipt_id', sa.UUID(), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('ide_build', sa.String(), nullable=False), + sa.Column('ai_assistant_build', sa.String(), nullable=False), + sa.Column('plugin_version', sa.String(), nullable=False), + sa.Column('os', sa.String(), nullable=False), + sa.Column('arch', sa.String(), nullable=False), + sa.Column('agent_id', sa.String(), nullable=False), + sa.Column('agent_version', sa.String(), nullable=False), + sa.Column('adapter_version', sa.String(), nullable=True), + sa.Column('protocol_version', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('content_hash', sa.String(), nullable=False), + sa.Column('receipt_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('receipt_id'), + sa.UniqueConstraint('content_hash'), + schema='public' + ) + op.create_index('idx_acp_capability_receipt_agent', 'acp_capability_receipt', ['agent_id', 'agent_version'], unique=False, schema='public') + op.create_index('idx_acp_capability_receipt_captured_at', 'acp_capability_receipt', ['captured_at'], unique=False, schema='public') + op.create_index('idx_acp_capability_receipt_status', 'acp_capability_receipt', ['status'], unique=False, schema='public') + op.create_table('agent_memory', + sa.Column('session_id', sa.String(), nullable=False), + sa.Column('owner_user_id', sa.String(), nullable=False), + sa.Column('owner_project_id', sa.String(), nullable=False), + sa.Column('memory_json', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('session_id'), + schema='public' + ) + op.create_index('idx_agent_memory_owner_project_id', 'agent_memory', ['owner_project_id'], unique=False, schema='public') + op.create_index('idx_agent_memory_owner_user_id', 'agent_memory', ['owner_user_id'], unique=False, schema='public') + op.create_index('idx_agent_memory_updated_at', 'agent_memory', ['updated_at'], unique=False, schema='public') + op.create_table('agent_release', + sa.Column('release_id', sa.String(), nullable=False), + sa.Column('agent_id', sa.String(), nullable=False), + sa.Column('source_manifest_digest', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('release_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('release_id'), + sa.UniqueConstraint('agent_id', 'source_manifest_digest', name='uq_agent_release_agent_manifest'), + schema='public' + ) + op.create_index('idx_agent_release_agent_id', 'agent_release', ['agent_id'], unique=False, schema='public') + op.create_index('idx_agent_release_status', 'agent_release', ['status'], unique=False, schema='public') + op.create_table('provider_connection', + sa.Column('connection_id', sa.UUID(), nullable=False), + sa.Column('label', sa.String(), nullable=False), + sa.Column('base_url', sa.String(), nullable=False), + sa.Column('secret_ref', sa.String(), nullable=False), + sa.Column('models_json', sa.Text(), nullable=False), + sa.Column('is_active', sa.Boolean(), server_default='true', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('connection_id'), + sa.UniqueConstraint('label'), + schema='public' + ) + op.create_index('idx_provider_connection_is_active', 'provider_connection', ['is_active'], unique=False, schema='public') + op.create_table('research_event', + sa.Column('event_id', sa.UUID(), nullable=False), + sa.Column('schema_version', sa.String(), nullable=False), + sa.Column('event_type', sa.String(), nullable=False), + sa.Column('source', sa.String(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('agent_run_id', sa.String(), nullable=True), + sa.Column('emitter_id', sa.String(), nullable=False), + sa.Column('emitter_sequence', sa.Integer(), nullable=False), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('envelope_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('digest', sa.String(), nullable=False), + sa.Column('accepted_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('retention_state', sa.String(), server_default='RETAINED', nullable=False), + sa.Column('anonymized_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('event_id'), + sa.UniqueConstraint('research_session_id', 'emitter_id', 'emitter_sequence', name='uq_research_event_session_emitter_sequence'), + schema='public' + ) + op.create_index('idx_research_event_digest', 'research_event', ['digest'], unique=False, schema='public') + op.create_index('idx_research_event_emitter_id', 'research_event', ['emitter_id'], unique=False, schema='public') + op.create_index('idx_research_event_enrollment_id', 'research_event', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_event_event_type', 'research_event', ['event_type'], unique=False, schema='public') + op.create_index('idx_research_event_occurred_at', 'research_event', ['occurred_at'], unique=False, schema='public') + op.create_index('idx_research_event_session_id', 'research_event', ['research_session_id'], unique=False, schema='public') + op.create_table('research_participant', + sa.Column('participant_id', sa.UUID(), nullable=False), + sa.Column('account_id', sa.UUID(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('participant_id'), + sa.UniqueConstraint('account_id'), + schema='public' + ) + op.create_index('idx_research_participant_account_id', 'research_participant', ['account_id'], unique=False, schema='public') + op.create_table('research_record', + sa.Column('record_id', sa.UUID(), nullable=False), + sa.Column('kind', sa.String(), nullable=False), + sa.Column('scope_type', sa.String(), nullable=True), + sa.Column('scope_id', sa.UUID(), nullable=True), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('actor', sa.String(), nullable=True), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('payload_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.PrimaryKeyConstraint('record_id'), + schema='public' + ) + op.create_index('idx_research_record_kind', 'research_record', ['kind'], unique=False, schema='public') + op.create_index('idx_research_record_occurred_at', 'research_record', ['occurred_at'], unique=False, schema='public') + op.create_index('idx_research_record_scope', 'research_record', ['scope_type', 'scope_id'], unique=False, schema='public') + op.create_index('idx_research_record_study_id', 'research_record', ['study_id'], unique=False, schema='public') + op.create_table('research_retention_job', + sa.Column('job_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('action', sa.String(), nullable=False), + sa.Column('state', sa.String(), nullable=False), + sa.Column('attempts', sa.Integer(), server_default='0', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_error', sa.String(), nullable=True), + sa.Column('evidence_digest', sa.String(), nullable=True), + sa.Column('affected_events', sa.Integer(), server_default='0', nullable=False), + sa.Column('evidence_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.PrimaryKeyConstraint('job_id'), + sa.UniqueConstraint('enrollment_id', 'action', name='uq_research_retention_job_enrollment_action'), + schema='public' + ) + op.create_index('idx_research_retention_job_enrollment', 'research_retention_job', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_retention_job_state', 'research_retention_job', ['state'], unique=False, schema='public') + op.create_table('research_session', + sa.Column('session_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('context_id', sa.String(), nullable=False), + sa.Column('state', sa.String(), nullable=False), + sa.Column('opened_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_activity_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('closed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('close_reason', sa.String(), nullable=True), + sa.Column('resume_generation', sa.Integer(), server_default='0', nullable=False), + sa.Column('manifest_digest', sa.String(), nullable=False), + sa.Column('environment_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('transitions_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('session_id'), + schema='public' + ) + op.create_index('idx_research_session_context_id', 'research_session', ['context_id'], unique=False, schema='public') + op.create_index('idx_research_session_enrollment_id', 'research_session', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_session_state', 'research_session', ['state'], unique=False, schema='public') + op.create_index('idx_research_session_study_id', 'research_session', ['study_id'], unique=False, schema='public') + op.create_index('uq_research_session_active_context', 'research_session', ['enrollment_id', 'context_id'], unique=True, schema='public', postgresql_where=sa.text("state NOT IN ('ended', 'revoked')")) + op.create_table('research_agent_run', + sa.Column('agent_run_id', sa.UUID(), nullable=False), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('agent_release_id', sa.String(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('ended_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('outcome', sa.String(), nullable=True), + sa.Column('snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('snapshot_captured_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['research_session_id'], ['public.research_session.session_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('agent_run_id'), + schema='public' + ) + op.create_index('idx_research_agent_run_session_id', 'research_agent_run', ['research_session_id'], unique=False, schema='public') + op.create_table('research_enrollment', + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('participant_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('participant_code', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('revocation_epoch', sa.Integer(), server_default='0', nullable=False), + sa.Column('eligibility_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('enrolled_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('consent_accepted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('retention_action', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['participant_id'], ['public.research_participant.participant_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('enrollment_id'), + sa.UniqueConstraint('participant_id', 'study_id', name='uq_research_enrollment_participant_study'), + schema='public' + ) + op.create_index('idx_research_enrollment_participant_code', 'research_enrollment', ['participant_code'], unique=False, schema='public') + op.create_index('idx_research_enrollment_participant_id', 'research_enrollment', ['participant_id'], unique=False, schema='public') + op.create_index('idx_research_enrollment_status', 'research_enrollment', ['status'], unique=False, schema='public') + op.create_index('idx_research_enrollment_study_id', 'research_enrollment', ['study_id'], unique=False, schema='public') + op.create_index('uq_research_enrollment_active_participant', 'research_enrollment', ['participant_id'], unique=True, schema='public', postgresql_where=sa.text("status = 'ACTIVE'")) + op.create_table('agent_profile', + sa.Column('profile_id', sa.UUID(), nullable=False), + sa.Column('owner_user_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('model', sa.String(), nullable=False), + sa.Column('framework_version', sa.String(), server_default='code4me2-agent', nullable=False), + sa.Column('release_id', sa.String(), nullable=True), + sa.Column('connection_id', sa.UUID(), nullable=True), + sa.Column('tools_json', sa.Text(), nullable=False), + sa.Column('approval_policy', sa.String(), nullable=False), + sa.Column('max_steps', sa.Integer(), nullable=False), + sa.Column('temperature', sa.Double(), nullable=True), + sa.Column('max_context_tokens', sa.Integer(), nullable=True), + sa.Column('is_active', sa.Boolean(), server_default='true', nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['connection_id'], ['public.provider_connection.connection_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['owner_user_id'], ['public.user.user_id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['release_id'], ['public.agent_release.release_id'], ondelete='RESTRICT'), + sa.PrimaryKeyConstraint('profile_id'), + sa.UniqueConstraint('owner_user_id', 'name', name='uq_agent_profile_owner_name'), + schema='public' + ) + op.create_index('idx_agent_profile_is_active', 'agent_profile', ['is_active'], unique=False, schema='public') + op.create_index('idx_agent_profile_owner_user_id', 'agent_profile', ['owner_user_id'], unique=False, schema='public') + op.create_table('study_assignment', + sa.Column('assignment_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('agent_profile_id', sa.UUID(), nullable=False), + sa.Column('strategy', sa.String(), nullable=False), + sa.Column('randomization_epoch', sa.Integer(), server_default='0', nullable=False), + sa.Column('profile_digest', sa.String(), nullable=False), + sa.Column('profile_snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('assigned_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.research_enrollment.enrollment_id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['agent_profile_id'], ['public.agent_profile.profile_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('assignment_id'), + sa.UniqueConstraint('enrollment_id', name='uq_study_assignment_enrollment'), + schema='public' + ) + op.create_index('idx_study_assignment_enrollment_id', 'study_assignment', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_study_assignment_study_id', 'study_assignment', ['study_id'], unique=False, schema='public') + op.create_index('idx_study_assignment_agent_profile_id', 'study_assignment', ['agent_profile_id'], unique=False, schema='public') + op.create_table('telemetry_batch_receipt', + sa.Column('receipt_id', sa.UUID(), nullable=False), + sa.Column('batch_id', sa.String(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('accepted_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('receipt_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.research_enrollment.enrollment_id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['research_session_id'], ['public.research_session.session_id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('receipt_id'), + sa.UniqueConstraint('batch_id'), + schema='public' + ) + op.create_index('idx_telemetry_batch_receipt_session_id', 'telemetry_batch_receipt', ['research_session_id'], unique=False, schema='public') + op.create_table('agent_task', + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('session_id', sa.UUID(), nullable=True), + sa.Column('source', sa.String(), server_default='plugin', nullable=False), + sa.Column('owner_user_id', sa.UUID(), nullable=True), + sa.Column('funding_owner_user_id', sa.UUID(), nullable=True), + sa.Column('owner_project_id', sa.UUID(), nullable=True), + sa.Column('external_run_id', sa.String(), nullable=True), + sa.Column('agent_session_id', sa.String(), nullable=True), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('study_assignment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('profile_id', sa.UUID(), nullable=True), + sa.Column('agent_profile', sa.String(), nullable=False), + sa.Column('model', sa.String(), nullable=False), + sa.Column('temperature', sa.Double(), nullable=True), + sa.Column('approval_policy', sa.String(), nullable=False), + sa.Column('tools_json', sa.Text(), nullable=False), + sa.Column('framework_version', sa.String(), nullable=True), + sa.Column('policy_snapshot', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('observed_framework_version', sa.String(), nullable=True), + sa.Column('observed_tools_json', sa.Text(), nullable=True), + sa.Column('consent_content_storage', sa.Boolean(), nullable=True), + sa.Column('next_event_index', sa.Integer(), server_default='0', nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('total_steps', sa.Integer(), nullable=True), + sa.Column('input_tokens', sa.Integer(), nullable=True), + sa.Column('output_tokens', sa.Integer(), nullable=True), + sa.Column('latest_event_id', sa.UUID(), nullable=True), + sa.Column('task_description', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['funding_owner_user_id'], ['public.user.user_id'], ondelete='SET NULL'), + # The cyclic latest_event_id FK is added explicitly after both tables exist + # (declared use_alter=True in the model, so it is not created inline here). + sa.ForeignKeyConstraint(['owner_user_id'], ['public.user.user_id'], ), + sa.ForeignKeyConstraint(['profile_id'], ['public.agent_profile.profile_id'], ), + sa.ForeignKeyConstraint(['session_id'], ['public.session.session_id'], ), + sa.ForeignKeyConstraint(['study_assignment_id'], ['public.study_assignment.assignment_id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ), + sa.PrimaryKeyConstraint('task_id'), + sa.UniqueConstraint('external_run_id'), + schema='public' + ) + op.create_index('idx_agent_task_enrollment_id', 'agent_task', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_agent_task_owner_user_id', 'agent_task', ['owner_user_id'], unique=False, schema='public') + op.create_index('idx_agent_task_profile_id', 'agent_task', ['profile_id'], unique=False, schema='public') + op.create_index('idx_agent_task_research_session_id', 'agent_task', ['research_session_id'], unique=False, schema='public') + op.create_index('idx_agent_task_session_id', 'agent_task', ['session_id'], unique=False, schema='public') + op.create_index('idx_agent_task_status', 'agent_task', ['status'], unique=False, schema='public') + op.create_index('idx_agent_task_study_assignment_id', 'agent_task', ['study_assignment_id'], unique=False, schema='public') + op.create_index('idx_agent_task_study_id', 'agent_task', ['study_id'], unique=False, schema='public') + op.create_table('agent_edit', + sa.Column('edit_id', sa.UUID(), nullable=False), + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('file_path', sa.Text(), nullable=False), + sa.Column('was_accepted', sa.Boolean(), nullable=True), + sa.Column('was_modified', sa.Boolean(), nullable=True), + sa.Column('decided_at', sa.DateTime(), nullable=True), + sa.Column('diff_text', sa.Text(), nullable=True), + sa.Column('edit_delta_json', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['task_id'], ['public.agent_task.task_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('edit_id'), + schema='public' + ) + op.create_index('idx_agent_edit_task_id', 'agent_edit', ['task_id'], unique=False, schema='public') + op.create_table('agent_event', + sa.Column('event_id', sa.UUID(), nullable=False), + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('event_index', sa.Integer(), nullable=False), + sa.Column('event_type', sa.String(), nullable=False), + sa.Column('source', sa.String(), nullable=True), + sa.Column('source_event_id', sa.String(), nullable=True), + sa.Column('schema_version', sa.String(), nullable=True), + sa.Column('latency_ms', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('span_id', sa.String(), nullable=True), + sa.Column('parent_span_id', sa.UUID(), nullable=True), + sa.Column('request_id', sa.String(), nullable=True), + sa.Column('chat_session_index', sa.Integer(), nullable=True), + sa.Column('model', sa.String(), nullable=True), + sa.Column('agent_profile', sa.String(), nullable=True), + sa.Column('streaming', sa.Boolean(), nullable=True), + sa.Column('message_count', sa.Integer(), nullable=True), + sa.Column('role_system_count', sa.Integer(), nullable=True), + sa.Column('role_user_count', sa.Integer(), nullable=True), + sa.Column('role_assistant_count', sa.Integer(), nullable=True), + sa.Column('role_tool_count', sa.Integer(), nullable=True), + sa.Column('tools_kept', sa.Integer(), nullable=True), + sa.Column('tools_stripped', sa.Integer(), nullable=True), + sa.Column('tool_names_requested', sa.ARRAY(sa.String()), nullable=True), + sa.Column('max_tokens', sa.Integer(), nullable=True), + sa.Column('prompt_tokens', sa.Integer(), nullable=True), + sa.Column('completion_tokens', sa.Integer(), nullable=True), + sa.Column('total_tokens', sa.Integer(), nullable=True), + sa.Column('finish_reason', sa.String(), nullable=True), + sa.Column('upstream_status', sa.Integer(), nullable=True), + sa.Column('step_index', sa.Integer(), nullable=True), + sa.Column('context_window_size_bytes', sa.Integer(), nullable=True), + sa.Column('active_file', sa.Text(), nullable=True), + sa.Column('first_message_hash', sa.String(), nullable=True), + sa.Column('chat_new_session_detected', sa.Boolean(), nullable=True), + sa.Column('experiment_tool_access_enabled', sa.Boolean(), nullable=True), + sa.Column('experiment_approval_policy', sa.String(), nullable=True), + sa.Column('first_system_message', sa.Text(), nullable=True), + sa.Column('last_user_message', sa.Text(), nullable=True), + sa.Column('response_text', sa.Text(), nullable=True), + sa.Column('tool_name', sa.String(), nullable=True), + sa.Column('tool_arguments_length', sa.Integer(), nullable=True), + sa.Column('tool_result_length', sa.Integer(), nullable=True), + sa.Column('tool_arguments', sa.Text(), nullable=True), + sa.Column('tool_result', sa.Text(), nullable=True), + sa.Column('extra_json', sa.Text(), nullable=True), + sa.Column('payload_json', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['task_id'], ['public.agent_task.task_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('event_id'), + sa.UniqueConstraint('task_id', 'event_index', name='uq_agent_event_task_event_index'), + sa.UniqueConstraint('task_id', 'source', 'source_event_id', name='uq_agent_event_source_identity'), + schema='public' + ) + op.create_index('idx_agent_event_created_at', 'agent_event', ['created_at'], unique=False, schema='public') + op.create_index('idx_agent_event_event_type', 'agent_event', ['event_type'], unique=False, schema='public') + op.create_index('idx_agent_event_task_id', 'agent_event', ['task_id'], unique=False, schema='public') + op.add_column('study', sa.Column('is_research', sa.Boolean(), server_default='false', nullable=False), schema='public') + op.add_column('study', sa.Column('research_status', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('research_config_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), schema='public') + op.add_column('study', sa.Column('research_config_digest', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('join_code', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('consent_locked_at', sa.DateTime(timezone=True), nullable=True), schema='public') + op.add_column('study', sa.Column('stopped_at', sa.DateTime(timezone=True), nullable=True), schema='public') + op.add_column('study', sa.Column('stopped_by', sa.String(), nullable=True), schema='public') + op.create_table('study_agent_profile', + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('profile_id', sa.UUID(), nullable=False), + sa.Column('profile_digest', sa.String(), nullable=False), + sa.Column('profile_snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('selection_order', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['profile_id'], ['public.agent_profile.profile_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('study_id', 'profile_id'), + sa.UniqueConstraint('study_id', 'profile_id', name='uq_study_agent_profile_selection'), + schema='public' + ) + op.create_index('idx_study_agent_profile_profile_id', 'study_agent_profile', ['profile_id'], unique=False, schema='public') + op.alter_column('study', 'default_config_id', + existing_type=sa.BIGINT(), + nullable=True, + schema='public') + op.create_index('uq_study_owner_live_research', 'study', ['created_by'], unique=True, schema='public', postgresql_where=sa.text('is_research AND is_active')) + op.create_index('idx_study_research_status', 'study', ['research_status'], unique=False, schema='public') + op.create_index('uq_study_research_join_code', 'study', ['join_code'], unique=True, schema='public', postgresql_where=sa.text('is_research AND join_code IS NOT NULL')) + op.add_column('user', sa.Column('can_research', sa.Boolean(), server_default='false', nullable=False), schema='public') + op.create_index('idx_user_can_research', 'user', ['can_research'], unique=False, schema='public') + # Cyclic FK (agent_task.latest_event_id -> agent_event.event_id) declared + # with use_alter=True in the model, so autogenerate skips it. Added by hand + # after both tables exist. + # Research referential integrity (M02). Ownership/enrollment/study are + # real relationships; telemetry uses SET NULL so retention policy, not a + # parent-row delete, decides when events disappear. + op.create_foreign_key('fk_research_participant_account', 'research_participant', 'user', ['account_id'], ['user_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_session_enrollment', 'research_session', 'research_enrollment', ['enrollment_id'], ['enrollment_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_session_study', 'research_session', 'study', ['study_id'], ['study_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_event_study', 'research_event', 'study', ['study_id'], ['study_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key('fk_research_event_enrollment', 'research_event', 'research_enrollment', ['enrollment_id'], ['enrollment_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key('fk_research_event_session', 'research_event', 'research_session', ['research_session_id'], ['session_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key( + 'agent_task_latest_event_id_fkey', + 'agent_task', + 'agent_event', + ['latest_event_id'], + ['event_id'], + source_schema='public', + referent_schema='public', + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + for _name, _table in ( + ('fk_research_event_session', 'research_event'), + ('fk_research_event_enrollment', 'research_event'), + ('fk_research_event_study', 'research_event'), + ('fk_research_session_study', 'research_session'), + ('fk_research_session_enrollment', 'research_session'), + ('fk_research_participant_account', 'research_participant'), + ): + op.drop_constraint(_name, _table, schema='public', type_='foreignkey') + """Downgrade database schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('idx_user_can_research', table_name='user', schema='public') + op.drop_column('user', 'can_research', schema='public') + op.drop_index('uq_study_research_join_code', table_name='study', schema='public', postgresql_where=sa.text('is_research AND join_code IS NOT NULL')) + op.drop_index('idx_study_research_status', table_name='study', schema='public') + op.drop_index('idx_study_agent_profile_profile_id', table_name='study_agent_profile', schema='public') + op.drop_table('study_agent_profile', schema='public') + op.drop_index('uq_study_owner_live_research', table_name='study', schema='public', postgresql_where=sa.text('is_research AND is_active')) + op.drop_column('study', 'stopped_by', schema='public') + op.drop_column('study', 'stopped_at', schema='public') + op.drop_column('study', 'consent_locked_at', schema='public') + op.drop_column('study', 'join_code', schema='public') + op.drop_column('study', 'research_config_digest', schema='public') + op.drop_column('study', 'research_config_json', schema='public') + op.drop_column('study', 'research_status', schema='public') + op.alter_column('study', 'default_config_id', + existing_type=sa.BIGINT(), + nullable=False, + schema='public') + op.drop_column('study', 'is_research', schema='public') + op.drop_index('idx_agent_event_task_id', table_name='agent_event', schema='public') + op.drop_index('idx_agent_event_event_type', table_name='agent_event', schema='public') + op.drop_index('idx_agent_event_created_at', table_name='agent_event', schema='public') + op.drop_constraint('agent_task_latest_event_id_fkey', 'agent_task', schema='public', type_='foreignkey') + op.drop_table('agent_event', schema='public') + op.drop_index('idx_agent_edit_task_id', table_name='agent_edit', schema='public') + op.drop_table('agent_edit', schema='public') + op.drop_index('idx_agent_task_study_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_study_assignment_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_status', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_session_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_research_session_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_profile_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_owner_user_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_enrollment_id', table_name='agent_task', schema='public') + op.drop_table('agent_task', schema='public') + op.drop_index('idx_telemetry_batch_receipt_session_id', table_name='telemetry_batch_receipt', schema='public') + op.drop_table('telemetry_batch_receipt', schema='public') + op.drop_index('idx_study_assignment_agent_profile_id', table_name='study_assignment', schema='public') + op.drop_index('idx_study_assignment_study_id', table_name='study_assignment', schema='public') + op.drop_index('idx_study_assignment_enrollment_id', table_name='study_assignment', schema='public') + op.drop_table('study_assignment', schema='public') + op.drop_index('idx_agent_profile_owner_user_id', table_name='agent_profile', schema='public') + op.drop_index('idx_agent_profile_is_active', table_name='agent_profile', schema='public') + op.drop_table('agent_profile', schema='public') + op.drop_index('uq_research_enrollment_active_participant', table_name='research_enrollment', schema='public', postgresql_where=sa.text("status = 'ACTIVE'")) + op.drop_index('idx_research_enrollment_study_id', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_status', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_participant_id', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_participant_code', table_name='research_enrollment', schema='public') + op.drop_table('research_enrollment', schema='public') + op.drop_index('idx_research_agent_run_session_id', table_name='research_agent_run', schema='public') + op.drop_table('research_agent_run', schema='public') + op.drop_index('uq_research_session_active_context', table_name='research_session', schema='public', postgresql_where=sa.text("state NOT IN ('ended', 'revoked')")) + op.drop_index('idx_research_session_study_id', table_name='research_session', schema='public') + op.drop_index('idx_research_session_state', table_name='research_session', schema='public') + op.drop_index('idx_research_session_enrollment_id', table_name='research_session', schema='public') + op.drop_index('idx_research_session_context_id', table_name='research_session', schema='public') + op.drop_table('research_session', schema='public') + op.drop_index('idx_research_retention_job_state', table_name='research_retention_job', schema='public') + op.drop_index('idx_research_retention_job_enrollment', table_name='research_retention_job', schema='public') + op.drop_table('research_retention_job', schema='public') + op.drop_index('idx_research_record_study_id', table_name='research_record', schema='public') + op.drop_index('idx_research_record_scope', table_name='research_record', schema='public') + op.drop_index('idx_research_record_occurred_at', table_name='research_record', schema='public') + op.drop_index('idx_research_record_kind', table_name='research_record', schema='public') + op.drop_table('research_record', schema='public') + op.drop_index('idx_research_participant_account_id', table_name='research_participant', schema='public') + op.drop_table('research_participant', schema='public') + op.drop_index('idx_research_event_session_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_occurred_at', table_name='research_event', schema='public') + op.drop_index('idx_research_event_event_type', table_name='research_event', schema='public') + op.drop_index('idx_research_event_enrollment_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_emitter_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_digest', table_name='research_event', schema='public') + op.drop_table('research_event', schema='public') + op.drop_index('idx_provider_connection_is_active', table_name='provider_connection', schema='public') + op.drop_table('provider_connection', schema='public') + op.drop_index('idx_agent_release_status', table_name='agent_release', schema='public') + op.drop_index('idx_agent_release_agent_id', table_name='agent_release', schema='public') + op.drop_table('agent_release', schema='public') + op.drop_index('idx_agent_memory_updated_at', table_name='agent_memory', schema='public') + op.drop_index('idx_agent_memory_owner_user_id', table_name='agent_memory', schema='public') + op.drop_index('idx_agent_memory_owner_project_id', table_name='agent_memory', schema='public') + op.drop_table('agent_memory', schema='public') + op.drop_index('idx_acp_capability_receipt_status', table_name='acp_capability_receipt', schema='public') + op.drop_index('idx_acp_capability_receipt_captured_at', table_name='acp_capability_receipt', schema='public') + op.drop_index('idx_acp_capability_receipt_agent', table_name='acp_capability_receipt', schema='public') + op.drop_table('acp_capability_receipt', schema='public') + # ### end Alembic commands ### \ No newline at end of file diff --git a/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py b/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py deleted file mode 100644 index db35a66..0000000 --- a/src/database/migration/versions/c2dc3e9cc1bc_baseline_migration_from_init_sql.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Baseline migration from init.sql - -Deliberately empty. A fresh database is created by executing ``init.sql`` -(see ``migration_manager.initialize_from_sql``); this revision exists only to -give that state a name so subsequent migrations have something to chain from. - -Revision ID: c2dc3e9cc1bc -Revises: -Create Date: 2026-09-06 - -""" - -# revision identifiers, used by Alembic. -revision = "c2dc3e9cc1bc" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - """Upgrade database schema.""" - pass - - -def downgrade() -> None: - """Downgrade database schema.""" - pass diff --git a/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py b/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py deleted file mode 100644 index ce02453..0000000 --- a/src/database/migration/versions/d1a2b3c4e5f7_add_agent_tables.py +++ /dev/null @@ -1,396 +0,0 @@ -"""add unified agent tables - -Creates the seven tables backing the agent subsystem in their final, -already-unified shape: - - agent_profile experiment variant: runtime + provider + tools + policy - agent_profile_assignment sticky per-user A/B arm - agent_task one row per agent session, config snapshotted - agent_event one row per step, "columnar core + JSON overflow" - agent_edit per-file proposal + accept/reject/modified + diff - agent_memory session-scoped memory surviving IDE restarts - study_agent_profile links the existing Study A/B framework to agent arms - -``agent_event`` gives typed columns to everything both telemetry paths produce -(the client-side proxy observing Goose/Codex, and the built-in code4me2-agent -runtime self-reporting), and routes adapter-specific extras into a single -``extra_json`` overflow column so a new adapter never requires a migration. - -Content columns (task_description, first_system_message, last_user_message, -response_text, tool_arguments, tool_result, payload_json, diff_text, -edit_delta_json) are all nullable: they are only ever written when the user's -``store_agent_content`` preference resolves True, enforced server-side. - -Revision ID: d1a2b3c4e5f7 -Revises: c2dc3e9cc1bc -Create Date: 2026-09-06 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "d1a2b3c4e5f7" -down_revision = "c2dc3e9cc1bc" -branch_labels = None -depends_on = None - -_SCHEMA = "public" - - -def upgrade() -> None: - op.create_table( - "agent_profile", - sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("model", sa.String(), nullable=False), - # Which agent runtime this profile targets: code4me2-agent | goose | codex - sa.Column( - "framework_version", - sa.String(), - nullable=False, - server_default=sa.text("'code4me2-agent'"), - ), - # Generic OpenAI-compatible provider triple. NULL base_url = server default. - sa.Column("base_url", sa.String(), nullable=True), - # Name of the env var holding the key — never the key itself. - sa.Column("api_key_ref", sa.String(), nullable=True), - sa.Column("tools_json", sa.Text(), nullable=False), - sa.Column("approval_policy", sa.String(), nullable=False), - sa.Column("max_steps", sa.Integer(), nullable=False), - sa.Column("temperature", sa.Double(), nullable=True), - sa.Column("max_context_tokens", sa.Integer(), nullable=True), - sa.Column( - "is_active", sa.Boolean(), nullable=False, server_default=sa.text("true") - ), - sa.Column("created_at", sa.DateTime(), nullable=True), - sa.PrimaryKeyConstraint("profile_id"), - sa.UniqueConstraint("name"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_profile_is_active", - "agent_profile", - ["is_active"], - unique=False, - schema=_SCHEMA, - ) - - op.create_table( - "agent_profile_assignment", - sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column( - "source", sa.String(), nullable=False, server_default=sa.text("'auto'") - ), - sa.Column( - "assigned_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["user_id"], ["public.user.user_id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint( - ["profile_id"], - ["public.agent_profile.profile_id"], - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("user_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_profile_assignment_profile_id", - "agent_profile_assignment", - ["profile_id"], - unique=False, - schema=_SCHEMA, - ) - - # agent_task and agent_event reference each other (agent_event.task_id → - # agent_task, agent_task.latest_event_id → agent_event), so the cycle is - # broken by adding latest_event_id's FK after both tables exist. - op.create_table( - "agent_task", - sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("session_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column( - "source", sa.String(), nullable=False, server_default=sa.text("'plugin'") - ), - sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("owner_project_id", postgresql.UUID(as_uuid=True), nullable=True), - # The runtime's own ids, which are not guaranteed to be UUID-shaped. - sa.Column("external_run_id", sa.String(), nullable=True), - sa.Column("agent_session_id", sa.String(), nullable=True), - sa.Column("agent_profile", sa.String(), nullable=False), - sa.Column("model", sa.String(), nullable=False), - sa.Column("temperature", sa.Double(), nullable=True), - sa.Column("approval_policy", sa.String(), nullable=False), - sa.Column("tools_json", sa.Text(), nullable=False), - sa.Column("framework_version", sa.String(), nullable=True), - sa.Column("status", sa.String(), nullable=False), - sa.Column("created_at", sa.DateTime(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(), nullable=True), - sa.Column("total_steps", sa.Integer(), nullable=True), - sa.Column("input_tokens", sa.Integer(), nullable=True), - sa.Column("output_tokens", sa.Integer(), nullable=True), - sa.Column("latest_event_id", postgresql.UUID(as_uuid=True), nullable=True), - # Content — gated by store_agent_content - sa.Column("task_description", sa.Text(), nullable=True), - sa.ForeignKeyConstraint(["session_id"], ["public.session.session_id"]), - sa.ForeignKeyConstraint(["owner_user_id"], ["public.user.user_id"]), - sa.PrimaryKeyConstraint("task_id"), - sa.UniqueConstraint("external_run_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_task_status", "agent_task", ["status"], unique=False, schema=_SCHEMA - ) - op.create_index( - "idx_agent_task_session_id", - "agent_task", - ["session_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_task_owner_user_id", - "agent_task", - ["owner_user_id"], - unique=False, - schema=_SCHEMA, - ) - - op.create_table( - "agent_event", - sa.Column("event_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("event_index", sa.Integer(), nullable=False), - sa.Column("event_type", sa.String(), nullable=False), - # "proxy" | "code4me2_agent" - sa.Column("source", sa.String(), nullable=True), - sa.Column("schema_version", sa.String(), nullable=True), - sa.Column("latency_ms", sa.Integer(), nullable=True), - sa.Column("created_at", sa.DateTime(), nullable=True), - sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=True), - # Span tree - sa.Column("span_id", sa.String(), nullable=True), - sa.Column("parent_span_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("request_id", sa.String(), nullable=True), - sa.Column("chat_session_index", sa.Integer(), nullable=True), - # model_call fields - sa.Column("model", sa.String(), nullable=True), - sa.Column("agent_profile", sa.String(), nullable=True), - sa.Column("streaming", sa.Boolean(), nullable=True), - sa.Column("message_count", sa.Integer(), nullable=True), - sa.Column("role_system_count", sa.Integer(), nullable=True), - sa.Column("role_user_count", sa.Integer(), nullable=True), - sa.Column("role_assistant_count", sa.Integer(), nullable=True), - sa.Column("role_tool_count", sa.Integer(), nullable=True), - sa.Column("tools_kept", sa.Integer(), nullable=True), - sa.Column("tools_stripped", sa.Integer(), nullable=True), - sa.Column("tool_names_requested", postgresql.ARRAY(sa.String()), nullable=True), - sa.Column("max_tokens", sa.Integer(), nullable=True), - sa.Column("prompt_tokens", sa.Integer(), nullable=True), - sa.Column("completion_tokens", sa.Integer(), nullable=True), - sa.Column("total_tokens", sa.Integer(), nullable=True), - sa.Column("finish_reason", sa.String(), nullable=True), - sa.Column("upstream_status", sa.Integer(), nullable=True), - sa.Column("step_index", sa.Integer(), nullable=True), - sa.Column("context_window_size_bytes", sa.Integer(), nullable=True), - sa.Column("active_file", sa.Text(), nullable=True), - sa.Column("first_message_hash", sa.String(), nullable=True), - sa.Column("chat_new_session_detected", sa.Boolean(), nullable=True), - sa.Column("experiment_tool_access_enabled", sa.Boolean(), nullable=True), - sa.Column("experiment_approval_policy", sa.String(), nullable=True), - # model_call content — gated by store_agent_content - sa.Column("first_system_message", sa.Text(), nullable=True), - sa.Column("last_user_message", sa.Text(), nullable=True), - sa.Column("response_text", sa.Text(), nullable=True), - # tool_call fields - sa.Column("tool_name", sa.String(), nullable=True), - sa.Column("tool_arguments_length", sa.Integer(), nullable=True), - sa.Column("tool_result_length", sa.Integer(), nullable=True), - # tool_call content — gated by store_agent_content - sa.Column("tool_arguments", sa.Text(), nullable=True), - sa.Column("tool_result", sa.Text(), nullable=True), - # JSON overflow: adapter-specific structural metrics (never content) - sa.Column("extra_json", sa.Text(), nullable=True), - # Free-form runtime payload — content, gated by store_agent_content - sa.Column("payload_json", sa.Text(), nullable=True), - sa.ForeignKeyConstraint( - ["task_id"], ["public.agent_task.task_id"], ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("event_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_event_task_id", - "agent_event", - ["task_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_event_event_type", - "agent_event", - ["event_type"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_event_created_at", - "agent_event", - ["created_at"], - unique=False, - schema=_SCHEMA, - ) - op.create_foreign_key( - "agent_task_latest_event_id_fkey", - "agent_task", - "agent_event", - ["latest_event_id"], - ["event_id"], - source_schema=_SCHEMA, - referent_schema=_SCHEMA, - ) - - op.create_table( - "agent_edit", - sa.Column("edit_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("file_path", sa.Text(), nullable=False), - sa.Column("was_accepted", sa.Boolean(), nullable=True), - sa.Column("was_modified", sa.Boolean(), nullable=True), - sa.Column("decided_at", sa.DateTime(), nullable=True), - # Content — gated by store_agent_content - sa.Column("diff_text", sa.Text(), nullable=True), - sa.Column("edit_delta_json", sa.Text(), nullable=True), - sa.ForeignKeyConstraint( - ["task_id"], ["public.agent_task.task_id"], ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("edit_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_edit_task_id", - "agent_edit", - ["task_id"], - unique=False, - schema=_SCHEMA, - ) - - op.create_table( - "agent_memory", - # ACP session ids are opaque strings, not guaranteed to be UUIDs. - sa.Column("session_id", sa.String(), nullable=False), - sa.Column("owner_user_id", sa.String(), nullable=False), - sa.Column("owner_project_id", sa.String(), nullable=False), - sa.Column("memory_json", sa.Text(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint("session_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_memory_owner_user_id", - "agent_memory", - ["owner_user_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_memory_owner_project_id", - "agent_memory", - ["owner_project_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_memory_updated_at", - "agent_memory", - ["updated_at"], - unique=False, - schema=_SCHEMA, - ) - - op.create_table( - "study_agent_profile", - sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column( - "is_baseline", sa.Boolean(), nullable=False, server_default=sa.text("false") - ), - sa.ForeignKeyConstraint( - ["study_id"], ["public.study.study_id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint(["profile_id"], ["public.agent_profile.profile_id"]), - sa.PrimaryKeyConstraint("study_id", "profile_id"), - schema=_SCHEMA, - ) - op.create_index( - "idx_study_agent_profile_profile_id", - "study_agent_profile", - ["profile_id"], - unique=False, - schema=_SCHEMA, - ) - - -def downgrade() -> None: - op.drop_index( - "idx_study_agent_profile_profile_id", - table_name="study_agent_profile", - schema=_SCHEMA, - ) - op.drop_table("study_agent_profile", schema=_SCHEMA) - - for index in ( - "idx_agent_memory_updated_at", - "idx_agent_memory_owner_project_id", - "idx_agent_memory_owner_user_id", - ): - op.drop_index(index, table_name="agent_memory", schema=_SCHEMA) - op.drop_table("agent_memory", schema=_SCHEMA) - - op.drop_index("idx_agent_edit_task_id", table_name="agent_edit", schema=_SCHEMA) - op.drop_table("agent_edit", schema=_SCHEMA) - - # Drop the cyclic FK before agent_event, so agent_event is droppable. - op.drop_constraint( - "agent_task_latest_event_id_fkey", - "agent_task", - type_="foreignkey", - schema=_SCHEMA, - ) - for index in ( - "idx_agent_event_created_at", - "idx_agent_event_event_type", - "idx_agent_event_task_id", - ): - op.drop_index(index, table_name="agent_event", schema=_SCHEMA) - op.drop_table("agent_event", schema=_SCHEMA) - - for index in ( - "idx_agent_task_owner_user_id", - "idx_agent_task_session_id", - "idx_agent_task_status", - ): - op.drop_index(index, table_name="agent_task", schema=_SCHEMA) - op.drop_table("agent_task", schema=_SCHEMA) - - op.drop_index( - "idx_agent_profile_assignment_profile_id", - table_name="agent_profile_assignment", - schema=_SCHEMA, - ) - op.drop_table("agent_profile_assignment", schema=_SCHEMA) - - op.drop_index( - "idx_agent_profile_is_active", table_name="agent_profile", schema=_SCHEMA - ) - op.drop_table("agent_profile", schema=_SCHEMA) diff --git a/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py b/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py deleted file mode 100644 index e9d14c6..0000000 --- a/src/database/migration/versions/e2b3c4d5f6a8_seed_default_agent_profiles.py +++ /dev/null @@ -1,120 +0,0 @@ -"""seed default agent profiles - -Without at least one *active* profile, POST /api/agent/task returns 503 ("No -active agent profiles are configured") and a freshly-migrated database cannot -mint tasks at all. This seeds one profile per supported runtime so the system -is usable out of the box. - -Only the built-in ``code4me2-agent`` profile is seeded active. For initial -testing it points at OpenRouter (an OpenAI-compatible endpoint, per -``agents.provider``) using the free ``cohere/north-mini-code:free`` model, with -``api_key_ref`` set to ``OPENROUTER_API_KEY`` — set that variable in ``.env`` -once a key is available; until then requests are forwarded unauthenticated and -will 401 against OpenRouter. The Goose and Codex profiles are seeded -*inactive* — they require a Goose binary / Node toolchain on the developer's -machine, so an operator flips them on from the admin UI once those -prerequisites exist. Inactive profiles are never drawn as A/B arms. - -No API keys are stored here (or anywhere in the DB): ``api_key_ref`` names the -environment variable to read the key from at request time. - -Revision ID: e2b3c4d5f6a8 -Revises: d1a2b3c4e5f7 -Create Date: 2026-09-06 - -""" - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "e2b3c4d5f6a8" -down_revision = "d1a2b3c4e5f7" -branch_labels = None -depends_on = None - -# Fixed ids so the seeded profiles are identical across all environments. -_PROFILES = [ - { - "profile_id": "7e1f20ce-fc5c-4ba2-b908-eaf77047d4b4", - "name": "default-code4me2-agent", - "framework_version": "code4me2-agent", - # OpenRouter speaks the OpenAI-compatible chat-completions format, so - # the generic provider path works against it unchanged. Using the free - # tier for initial testing; swap to a paid model once validated. - "base_url": "https://openrouter.ai/api/v1", - "api_key_ref": "OPENROUTER_API_KEY", - "model": "cohere/north-mini-code:free", - "tools_json": ( - '["read_file", "write_file", "create_file", "replace_text", ' - '"list_files", "search_files", "run_command"]' - ), - "approval_policy": "per_step", - "max_steps": 8, - "is_active": "true", - }, - { - "profile_id": "2b9d7c41-5a30-4c8e-9f21-1d6b0c3a7e58", - "name": "default-goose", - "framework_version": "goose", - "base_url": None, - "api_key_ref": "AGENT_UPSTREAM_API_KEY", - "model": "openai/gpt-oss-120b", - # Empty allowlist = the proxy forwards no tools. Populate from - # GET /api/agent/available-tools once a run has shown which tools the - # installed Goose build actually advertises. - "tools_json": "[]", - "approval_policy": "auto", - "max_steps": 15, - "is_active": "false", - }, - { - "profile_id": "6c4a8e12-93bf-4d57-8a0e-7f2c5b1d94a3", - "name": "default-codex", - "framework_version": "codex", - "base_url": "https://api.openai.com/v1", - "api_key_ref": "OPENAI_API_KEY", - "model": "gpt-5.1-codex-mini", - # Codex manages its own tool schemas over the Responses API; the proxy - # passes them through rather than filtering against an allowlist. - "tools_json": "[]", - "approval_policy": "auto", - "max_steps": 15, - "is_active": "false", - }, -] - - -def upgrade() -> None: - """Seed one profile per runtime. - - Idempotent: each insert is skipped when a profile of that name already - exists (e.g. created via the management website or a manual dev seed), so it - never collides on the unique name or the primary key. - """ - for profile in _PROFILES: - base_url = ( - "NULL" if profile["base_url"] is None else f"'{profile['base_url']}'" - ) - op.execute( - f""" - INSERT INTO public.agent_profile - (profile_id, name, model, framework_version, base_url, api_key_ref, - tools_json, approval_policy, max_steps, temperature, - max_context_tokens, is_active, created_at) - SELECT '{profile["profile_id"]}', '{profile["name"]}', - '{profile["model"]}', '{profile["framework_version"]}', - {base_url}, '{profile["api_key_ref"]}', - '{profile["tools_json"]}', '{profile["approval_policy"]}', - {profile["max_steps"]}, NULL, NULL, {profile["is_active"]}, now() - WHERE NOT EXISTS ( - SELECT 1 FROM public.agent_profile WHERE name = '{profile["name"]}' - ); - """ - ) - - -def downgrade() -> None: - """Remove only the rows this migration could have inserted (by their fixed - ids), so profiles created by other means are left untouched.""" - ids = ", ".join(f"'{profile['profile_id']}'" for profile in _PROFILES) - op.execute(f"DELETE FROM public.agent_profile WHERE profile_id IN ({ids});") diff --git a/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py b/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py deleted file mode 100644 index eed7172..0000000 --- a/src/database/migration/versions/f3c4d5e6a7b9_add_managed_agent_policy_snapshot.py +++ /dev/null @@ -1,27 +0,0 @@ -"""add managed agent policy snapshot - -Revision ID: f3c4d5e6a7b9 -Revises: e2b3c4d5f6a8 -Create Date: 2026-09-10 -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -revision = "f3c4d5e6a7b9" -down_revision = "e2b3c4d5f6a8" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "agent_task", - sa.Column("policy_snapshot", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - schema="public", - ) - - -def downgrade() -> None: - op.drop_column("agent_task", "policy_snapshot", schema="public") diff --git a/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py b/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py deleted file mode 100644 index 3509928..0000000 --- a/src/database/migration/versions/f3c4d5e6f7a9_add_experiment_execution_integrity.py +++ /dev/null @@ -1,328 +0,0 @@ -"""add experiment execution integrity - -Adds study-scoped agent assignments and links new agent tasks to the exact -study arm, profile, and consent decision used at creation time. Task rows keep -their existing copied configuration as the immutable execution snapshot. Event -writes gain a task-local atomic index allocator and database-enforced -identity/order constraints. - -Legacy assignments and tasks are intentionally retained without inferred study -attribution. Existing events retain a NULL source_event_id because their source -identity cannot be reconstructed safely. - -Revision ID: f3c4d5e6f7a9 -Revises: f3c4d5e6a7b9 -Create Date: 2026-09-09 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - - -revision = "f3c4d5e6f7a9" -down_revision = "f3c4d5e6a7b9" -branch_labels = None -depends_on = None - -_SCHEMA = "public" - - -def upgrade() -> None: - op.create_table( - "agent_study_assignment", - sa.Column("assignment_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("arm_name", sa.String(), nullable=False), - sa.Column( - "is_baseline", sa.Boolean(), nullable=False, server_default=sa.text("false") - ), - sa.Column( - "source", sa.String(), nullable=False, server_default=sa.text("'auto'") - ), - sa.Column( - "assigned_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["study_id"], ["public.study.study_id"], ondelete="RESTRICT" - ), - sa.ForeignKeyConstraint( - ["user_id"], ["public.user.user_id"], ondelete="RESTRICT" - ), - sa.ForeignKeyConstraint( - ["profile_id"], - ["public.agent_profile.profile_id"], - ondelete="RESTRICT", - ), - sa.PrimaryKeyConstraint("assignment_id"), - sa.UniqueConstraint("study_id", "user_id", name="uq_agent_study_assignment"), - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_study_assignment_study_id", - "agent_study_assignment", - ["study_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_study_assignment_profile_id", - "agent_study_assignment", - ["profile_id"], - unique=False, - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("study_id", postgresql.UUID(as_uuid=True), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("study_assignment_id", postgresql.UUID(as_uuid=True), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("profile_id", postgresql.UUID(as_uuid=True), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("study_arm_name", sa.String(), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("study_arm_is_baseline", sa.Boolean(), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("consent_content_storage", sa.Boolean(), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("observed_framework_version", sa.String(), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column("observed_tools_json", sa.Text(), nullable=True), - schema=_SCHEMA, - ) - op.add_column( - "agent_task", - sa.Column( - "next_event_index", - sa.Integer(), - nullable=False, - server_default=sa.text("0"), - ), - schema=_SCHEMA, - ) - op.create_foreign_key( - "agent_task_study_id_fkey", - "agent_task", - "study", - ["study_id"], - ["study_id"], - source_schema=_SCHEMA, - referent_schema=_SCHEMA, - ) - op.create_foreign_key( - "agent_task_study_assignment_id_fkey", - "agent_task", - "agent_study_assignment", - ["study_assignment_id"], - ["assignment_id"], - source_schema=_SCHEMA, - referent_schema=_SCHEMA, - ondelete="SET NULL", - ) - op.create_foreign_key( - "agent_task_profile_id_fkey", - "agent_task", - "agent_profile", - ["profile_id"], - ["profile_id"], - source_schema=_SCHEMA, - referent_schema=_SCHEMA, - ) - op.create_index( - "idx_agent_task_study_id", - "agent_task", - ["study_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_task_study_assignment_id", - "agent_task", - ["study_assignment_id"], - unique=False, - schema=_SCHEMA, - ) - op.create_index( - "idx_agent_task_profile_id", - "agent_task", - ["profile_id"], - unique=False, - schema=_SCHEMA, - ) - op.execute( - """ - CREATE FUNCTION public.prevent_agent_task_snapshot_mutation() - RETURNS trigger AS $$ - BEGIN - IF OLD.profile_id IS NOT NULL AND ( - NEW.study_id IS DISTINCT FROM OLD.study_id OR - NEW.study_assignment_id IS DISTINCT FROM OLD.study_assignment_id OR - NEW.profile_id IS DISTINCT FROM OLD.profile_id OR - NEW.study_arm_name IS DISTINCT FROM OLD.study_arm_name OR - NEW.study_arm_is_baseline IS DISTINCT FROM OLD.study_arm_is_baseline OR - NEW.consent_content_storage IS DISTINCT FROM OLD.consent_content_storage OR - NEW.agent_profile IS DISTINCT FROM OLD.agent_profile OR - NEW.model IS DISTINCT FROM OLD.model OR - NEW.temperature IS DISTINCT FROM OLD.temperature OR - NEW.approval_policy IS DISTINCT FROM OLD.approval_policy OR - NEW.tools_json IS DISTINCT FROM OLD.tools_json OR - NEW.framework_version IS DISTINCT FROM OLD.framework_version OR - NEW.policy_snapshot IS DISTINCT FROM OLD.policy_snapshot - ) THEN - RAISE EXCEPTION 'Agent task execution snapshots are immutable'; - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER agent_task_snapshot_immutable - BEFORE UPDATE ON public.agent_task - FOR EACH ROW - EXECUTE FUNCTION public.prevent_agent_task_snapshot_mutation(); - """ - ) - - op.execute( - """ - UPDATE public.agent_task task - SET next_event_index = COALESCE( - ( - SELECT MAX(event.event_index) + 1 - FROM public.agent_event event - WHERE event.task_id = task.task_id - ), - 0 - ) - """ - ) - - op.add_column( - "agent_event", - sa.Column("source_event_id", sa.String(), nullable=True), - schema=_SCHEMA, - ) - - # Historical duplicate positions have no defensible automatic repair. Stop - # rather than silently selecting an arbitrary event as the canonical one. - op.execute( - """ - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 - FROM public.agent_event - GROUP BY task_id, event_index - HAVING COUNT(*) > 1 - ) THEN - RAISE EXCEPTION - 'Cannot add agent event ordering constraint: duplicate task/event indexes exist'; - END IF; - END $$; - """ - ) - op.create_unique_constraint( - "uq_agent_event_task_event_index", - "agent_event", - ["task_id", "event_index"], - schema=_SCHEMA, - ) - op.create_unique_constraint( - "uq_agent_event_source_identity", - "agent_event", - ["task_id", "source", "source_event_id"], - schema=_SCHEMA, - ) - - -def downgrade() -> None: - op.execute("DROP TRIGGER agent_task_snapshot_immutable ON public.agent_task") - op.execute("DROP FUNCTION public.prevent_agent_task_snapshot_mutation()") - op.drop_constraint( - "uq_agent_event_source_identity", - "agent_event", - type_="unique", - schema=_SCHEMA, - ) - op.drop_constraint( - "uq_agent_event_task_event_index", - "agent_event", - type_="unique", - schema=_SCHEMA, - ) - op.drop_column("agent_event", "source_event_id", schema=_SCHEMA) - - op.drop_index( - "idx_agent_task_study_assignment_id", table_name="agent_task", schema=_SCHEMA - ) - op.drop_index("idx_agent_task_profile_id", table_name="agent_task", schema=_SCHEMA) - op.drop_index("idx_agent_task_study_id", table_name="agent_task", schema=_SCHEMA) - op.drop_constraint( - "agent_task_profile_id_fkey", - "agent_task", - type_="foreignkey", - schema=_SCHEMA, - ) - op.drop_constraint( - "agent_task_study_assignment_id_fkey", - "agent_task", - type_="foreignkey", - schema=_SCHEMA, - ) - op.drop_constraint( - "agent_task_study_id_fkey", - "agent_task", - type_="foreignkey", - schema=_SCHEMA, - ) - for column_name in ( - "next_event_index", - "consent_content_storage", - "study_arm_is_baseline", - "study_arm_name", - "profile_id", - "study_assignment_id", - "study_id", - "observed_tools_json", - "observed_framework_version", - ): - op.drop_column("agent_task", column_name, schema=_SCHEMA) - - op.drop_index( - "idx_agent_study_assignment_profile_id", - table_name="agent_study_assignment", - schema=_SCHEMA, - ) - op.drop_index( - "idx_agent_study_assignment_study_id", - table_name="agent_study_assignment", - schema=_SCHEMA, - ) - op.drop_table("agent_study_assignment", schema=_SCHEMA) diff --git a/src/database/research_schemas.py b/src/database/research_schemas.py new file mode 100644 index 0000000..b3acf66 --- /dev/null +++ b/src/database/research_schemas.py @@ -0,0 +1,549 @@ +"""SQLAlchemy ORM models for the research platform stores. + +The research platform keeps its own revisioned namespace so the operational +agent tables stay independent. Data that is a pure projection of an existing +JSON document lives inside that document rather than in a child table: + +* capability evidence is ``acp_capability_receipt.receipt_json.evidence[]``; +* agent artifacts/adapter are ``agent_release.release_json.artifacts[]``/``.adapter``; +* runtime packaging evidence is ``agent_release.release_json.package_json`` and + its conformance receipts are ``agent_release.release_json.conformance[]``; +* a capability snapshot is ``research_agent_run.snapshot_json``; +* conditions are ``study_revision.protocol_json.conditions``; +* study identity is ``study_revision.study_json``; +* drafts are ``study_revision.status = DRAFT``; +* an emitter cursor is ``max(research_event.emitter_sequence)``; +* rejections are ``telemetry_batch_receipt.receipt_json.rejected[]``; +* session transitions are appended to ``research_session.transitions_json``; +* a withdrawal is ``research_enrollment.withdrawal_json``; +* retention evidence is ``research_retention_job.evidence_json``. + +Two cross-cutting concerns get a generic table: ``research_record`` for every +append-only audit/evidence family (kill switch, health, release evidence, pilot +run, publication, export, retention). Coverage/metric projections are derived +purely from canonical events and are not persisted. +""" + +from datetime import datetime + +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Index, + Integer, + String, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from .db import Base + + +# NOTE: ``os`` is used as a column attribute name because it is part of the +# published environment tuple. It shadows the stdlib module only inside this +# class scope, which is safe because the module never imports ``os``. +class AcpCapabilityReceipt(Base): + """One version-pinned, redacted capability receipt.""" + + __tablename__ = "acp_capability_receipt" + __table_args__ = ( + Index("idx_acp_capability_receipt_agent", "agent_id", "agent_version"), + Index("idx_acp_capability_receipt_status", "status"), + Index("idx_acp_capability_receipt_captured_at", "captured_at"), + {"schema": "public"}, + ) + + receipt_id = Column(UUID(as_uuid=True), primary_key=True) + captured_at = Column(DateTime(timezone=True), nullable=False) + ide_build = Column(String, nullable=False) + ai_assistant_build = Column(String, nullable=False) + plugin_version = Column(String, nullable=False) + os = Column(String, nullable=False) + arch = Column(String, nullable=False) + agent_id = Column(String, nullable=False) + agent_version = Column(String, nullable=False) + adapter_version = Column(String, nullable=True) + protocol_version = Column(String, nullable=False) + status = Column(String, nullable=False) + # Unique: a receipt's canonical content is its identity, so storing the same + # bytes twice is a duplicate rather than a new observation. + content_hash = Column(String, nullable=False, unique=True) + receipt_json = Column(JSONB, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 02: versioned study protocol and publication +# --------------------------------------------------------------------------- + + +class StudyAgentProfile(Base): + """One profile selected by a study at creation time. + + The digest and non-secret snapshot keep a DRAFT study stable if the shared + profile is edited before the study receives its first consent. Assignment + rows take their own snapshot when a participant is enrolled. + """ + + __tablename__ = "study_agent_profile" + __table_args__ = ( + Index("idx_study_agent_profile_profile_id", "profile_id"), + UniqueConstraint( + "study_id", "profile_id", name="uq_study_agent_profile_selection" + ), + {"schema": "public"}, + ) + + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + primary_key=True, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), + primary_key=True, + ) + profile_digest = Column(String, nullable=False) + profile_snapshot_json = Column(JSONB, nullable=False) + selection_order = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 04: agent registry and capability contract +# --------------------------------------------------------------------------- + + +class AgentRelease(Base): + """One digest-pinned agent release record. + + Identity is ``(agent_id, release_id, source_manifest_digest)``: a changed + upstream manifest digest is a distinct release, not an in-place update. The + unique constraint on ``(agent_id, source_manifest_digest)`` additionally + makes the same source bytes impossible to register twice. + """ + + __tablename__ = "agent_release" + __table_args__ = ( + Index("idx_agent_release_agent_id", "agent_id"), + Index("idx_agent_release_status", "status"), + UniqueConstraint( + "agent_id", + "source_manifest_digest", + name="uq_agent_release_agent_manifest", + ), + {"schema": "public"}, + ) + + release_id = Column(String, primary_key=True) + agent_id = Column(String, nullable=False) + source_manifest_digest = Column(String, nullable=False) + status = Column(String, nullable=False) + release_json = Column(JSONB, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 03: participant identity, consent and withdrawal +# --------------------------------------------------------------------------- + + +class ResearchParticipant(Base): + """Private account-to-participant mapping. + + This is the only research table that links a login account to a study + participant. It is privileged and is never returned to researchers or + embedded in an export. + """ + + __tablename__ = "research_participant" + __table_args__ = ( + Index("idx_research_participant_account_id", "account_id"), + {"schema": "public"}, + ) + + participant_id = Column(UUID(as_uuid=True), primary_key=True) + account_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class ResearchEnrollment(Base): + """One participant's study enrollment and consent identity.""" + + __tablename__ = "research_enrollment" + __table_args__ = ( + Index("idx_research_enrollment_participant_id", "participant_id"), + Index("idx_research_enrollment_study_id", "study_id"), + Index("idx_research_enrollment_participant_code", "participant_code"), + Index("idx_research_enrollment_status", "status"), + UniqueConstraint( + "participant_id", + "study_id", + name="uq_research_enrollment_participant_study", + ), + # One active enrollment per account across the whole platform. ACTIVE is + # the only state that holds the slot; a completed study frees it. + Index( + "uq_research_enrollment_active_participant", + "participant_id", + unique=True, + postgresql_where=text("status = 'ACTIVE'"), + ), + {"schema": "public"}, + ) + + enrollment_id = Column(UUID(as_uuid=True), primary_key=True) + participant_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_participant.participant_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column(UUID(as_uuid=True), nullable=False) + # Random study-local pseudonym. Never derived from the account id. + participant_code = Column(String, nullable=False) + status = Column(String, nullable=False) + revocation_epoch = Column(Integer, nullable=False, server_default="0", default=0) + eligibility_json = Column(JSONB, nullable=False) + enrolled_at = Column(DateTime(timezone=True), nullable=False) + updated_at = Column(DateTime(timezone=True), nullable=False) + # The single consent acceptance: set once when the participant joins. There + # is no document identity, re-consent or withdrawal. + consent_accepted_at = Column(DateTime(timezone=True), nullable=True) + retention_action = Column(String, nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 05: enrollment assignment, exposure and bootstrap capabilities +# --------------------------------------------------------------------------- + + +class StudyAssignment(Base): + """The immutable, sticky agent profile assigned to one enrollment.""" + + __tablename__ = "study_assignment" + __table_args__ = ( + Index("idx_study_assignment_enrollment_id", "enrollment_id"), + Index("idx_study_assignment_study_id", "study_id"), + Index("idx_study_assignment_agent_profile_id", "agent_profile_id"), + UniqueConstraint("enrollment_id", name="uq_study_assignment_enrollment"), + {"schema": "public"}, + ) + + assignment_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + nullable=False, + ) + agent_profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), + nullable=False, + ) + strategy = Column(String, nullable=False, default="RANDOM_EQUAL") + randomization_epoch = Column(Integer, nullable=False, server_default="0", default=0) + profile_digest = Column(String, nullable=False) + profile_snapshot_json = Column(JSONB, nullable=False) + status = Column(String, nullable=False, default="ACTIVE") + assigned_at = Column(DateTime(timezone=True), nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 07: research session lifecycle and agent runs +# --------------------------------------------------------------------------- + + +class ResearchSessionV1(Base): + """One research session: the authoritative participant-activity boundary. + + Idle/resume timing lives in the study configuration, never in these columns. + + ``context_id`` is an opaque, client-scoped execution-context identifier (a + project/window instance). The partial unique index + ``uq_research_session_active_context`` enforces at most one non-terminal + session per ``(enrollment_id, context_id)``, so creation is idempotent for a + context while different windows/projects get distinct sessions. It is never a + filesystem path or an account identifier. + """ + + __tablename__ = "research_session" + __table_args__ = ( + Index("idx_research_session_enrollment_id", "enrollment_id"), + Index("idx_research_session_study_id", "study_id"), + Index("idx_research_session_state", "state"), + Index("idx_research_session_context_id", "context_id"), + Index( + "uq_research_session_active_context", + "enrollment_id", + "context_id", + unique=True, + postgresql_where=text("state NOT IN ('ended', 'revoked')"), + ), + {"schema": "public"}, + ) + + session_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + nullable=False, + ) + # Opaque execution-context id (project/window instance), never a path. + context_id = Column(String, nullable=False) + state = Column(String, nullable=False) + opened_at = Column(DateTime(timezone=True), nullable=True) + last_activity_at = Column(DateTime(timezone=True), nullable=True) + closed_at = Column(DateTime(timezone=True), nullable=True) + close_reason = Column(String, nullable=True) + resume_generation = Column(Integer, nullable=False, server_default="0", default=0) + manifest_digest = Column(String, nullable=False) + environment_json = Column(JSONB, nullable=False, default=dict) + # Append-only transition log (one serialized transition per entry). It lives + # on the session row because every transition belongs to exactly one session. + transitions_json = Column(JSONB, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class ResearchAgentRun(Base): + """One agent process (or one qualification run) against a release. + + A capability snapshot belongs to the run that captured it, so the snapshot + JSON and its capture time live on this row. A qualification snapshot may be + uploaded before any participant session exists, so ``research_session_id`` + is nullable: a run with no session is a detached qualification run. + """ + + __tablename__ = "research_agent_run" + __table_args__ = ( + Index("idx_research_agent_run_session_id", "research_session_id"), + {"schema": "public"}, + ) + + agent_run_id = Column(UUID(as_uuid=True), primary_key=True) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="CASCADE"), + nullable=True, + ) + agent_release_id = Column(String, nullable=True) + started_at = Column(DateTime(timezone=True), nullable=False) + ended_at = Column(DateTime(timezone=True), nullable=True) + outcome = Column(String, nullable=True) + # The full ``CapabilitySnapshotV1`` JSON (declared-vs-observed) captured by + # this run, and when it was captured (nullable: not every run snapshots). + snapshot_json = Column(JSONB, nullable=True) + snapshot_captured_at = Column(DateTime(timezone=True), nullable=True) + + +# --------------------------------------------------------------------------- +# Issue 09: idempotent telemetry ingestion and research persistence +# --------------------------------------------------------------------------- + + +class ResearchEvent(Base): + """One immutable accepted canonical telemetry fact. + + ``event_id`` is globally unique; identity is additionally constrained by + ``(research_session_id, emitter_id, emitter_sequence)`` where a session + applies, so per-emitter ordering is preserved. Accepted payload bytes and + provenance are never updated in place. + """ + + __tablename__ = "research_event" + __table_args__ = ( + Index("idx_research_event_enrollment_id", "enrollment_id"), + Index("idx_research_event_session_id", "research_session_id"), + Index("idx_research_event_emitter_id", "emitter_id"), + Index("idx_research_event_occurred_at", "occurred_at"), + Index("idx_research_event_event_type", "event_type"), + Index("idx_research_event_digest", "digest"), + UniqueConstraint( + "research_session_id", + "emitter_id", + "emitter_sequence", + name="uq_research_event_session_emitter_sequence", + ), + {"schema": "public"}, + ) + + event_id = Column(UUID(as_uuid=True), primary_key=True) + schema_version = Column(String, nullable=False) + event_type = Column(String, nullable=False) + source = Column(String, nullable=False) + # SET NULL: telemetry is retained/deleted by the enrollment's retention + # policy, so a removed session/enrollment row must not cascade into the + # events. `agent_run_id` stays a plain string: it is an external run + # identifier, not `research_agent_run.agent_run_id` (a UUID). + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="SET NULL"), + nullable=True, + ) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="SET NULL"), + nullable=True, + ) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="SET NULL"), + nullable=True, + ) + agent_run_id = Column(String, nullable=True) + emitter_id = Column(String, nullable=False) + emitter_sequence = Column(Integer, nullable=False) + occurred_at = Column(DateTime(timezone=True), nullable=False) + # The complete validated CanonicalEventV1 envelope (the event authority). + # The scalar columns are searchable projections derived from it at insert + # time (identity, ordering and joins); content lives only in the envelope. + envelope_json = Column(JSONB, nullable=False, default=dict) + digest = Column(String, nullable=False) + accepted_at = Column(DateTime(timezone=True), nullable=False) + retention_state = Column(String, nullable=False, server_default="RETAINED") + # When a retention action anonymized or tombstoned this row (additive). + anonymized_at = Column(DateTime(timezone=True), nullable=True) + + +class TelemetryBatchReceipt(Base): + """An immutable, retry-safe batch acknowledgement record.""" + + __tablename__ = "telemetry_batch_receipt" + __table_args__ = ( + Index("idx_telemetry_batch_receipt_session_id", "research_session_id"), + {"schema": "public"}, + ) + + receipt_id = Column(UUID(as_uuid=True), primary_key=True) + batch_id = Column(String, nullable=False, unique=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="SET NULL"), + nullable=True, + ) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="SET NULL"), + nullable=True, + ) + accepted_at = Column(DateTime(timezone=True), nullable=False) + receipt_json = Column(JSONB, nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 12: researcher control plane, coverage and export +# +# Researcher enablement is a single ``user.can_research`` flag (P3); there is no +# per-study ``research_researcher_role`` table. +# --------------------------------------------------------------------------- + + +#: ``kind`` values for :class:`ResearchRecord`. +RECORD_KIND_KILL_SWITCH = "KILL_SWITCH" +RECORD_KIND_HEALTH = "HEALTH" +RECORD_KIND_RELEASE_EVIDENCE = "RELEASE_EVIDENCE" +RECORD_KIND_PILOT_RUN = "PILOT_RUN" +RECORD_KIND_STUDY_PUBLICATION = "STUDY_PUBLICATION" +RECORD_KIND_RETENTION_EVIDENCE = "RETENTION_EVIDENCE" + + +class ResearchRecord(Base): + """One generic append-only research record. + + Consolidates every cross-cutting evidence record into a single table: + kill-switch engagements, operational health, release evidence, pilot runs, + study publication/lifecycle audits, export audits and retention evidence + (deletion ledger + deletion-drill verification). ``kind`` selects the + family, ``scope_type``/``scope_id`` name the scoped subject (study, + revision, enrollment, export) and the full original record lives in + ``payload_json``. ``study_id``/``actor`` are nullable because export and + retention evidence is not always study- or actor-scoped. + """ + + __tablename__ = "research_record" + __table_args__ = ( + Index("idx_research_record_kind", "kind"), + Index("idx_research_record_scope", "scope_type", "scope_id"), + Index("idx_research_record_study_id", "study_id"), + Index("idx_research_record_occurred_at", "occurred_at"), + {"schema": "public"}, + ) + + record_id = Column(UUID(as_uuid=True), primary_key=True) + kind = Column(String, nullable=False) + scope_type = Column(String, nullable=True) + scope_id = Column(UUID(as_uuid=True), nullable=True) + study_id = Column(UUID(as_uuid=True), nullable=True) + actor = Column(String, nullable=True) + occurred_at = Column(DateTime(timezone=True), nullable=False) + payload_json = Column(JSONB, nullable=False, default=dict) + + +# --------------------------------------------------------------------------- +# Issue 13: pilot, operations, retention and the release gate +# --------------------------------------------------------------------------- + + +class ResearchRetentionJob(Base): + """One idempotent retention-execution job over an enrollment's stored data. + + The unique ``(enrollment_id, action)`` constraint makes withdrawal enqueue + idempotent: re-withdrawing never creates a second job. ``enrollment_id`` is + intentionally not a foreign key so the evidence survives even if an + enrollment row is later removed. + """ + + __tablename__ = "research_retention_job" + __table_args__ = ( + Index("idx_research_retention_job_enrollment", "enrollment_id"), + Index("idx_research_retention_job_state", "state"), + UniqueConstraint( + "enrollment_id", + "action", + name="uq_research_retention_job_enrollment_action", + ), + {"schema": "public"}, + ) + + job_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column(UUID(as_uuid=True), nullable=False) + action = Column(String, nullable=False) + state = Column(String, nullable=False) + attempts = Column(Integer, nullable=False, server_default="0", default=0) + created_at = Column(DateTime(timezone=True), nullable=False) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + last_error = Column(String, nullable=True) + evidence_digest = Column(String, nullable=True) + affected_events = Column(Integer, nullable=False, server_default="0", default=0) + # Retained ledger/verification evidence (and its digest) for this job. It + # lives here so the evidence survives the enrollment row it describes. + evidence_json = Column(JSONB, nullable=False, default=dict) + + +# --------------------------------------------------------------------------- +# Issue 11: runtime packaging and agent conformance +# +# Packaging evidence is not a table: a release's ``RuntimeManifestV2`` lives in +# ``agent_release.release_json.package_json`` and its immutable conformance +# receipts live in ``agent_release.release_json.conformance[]``. +# --------------------------------------------------------------------------- + + diff --git a/src/main.py b/src/main.py index d81af84..fc66fb2 100644 --- a/src/main.py +++ b/src/main.py @@ -17,6 +17,7 @@ """ import logging +import os import threading import time from contextlib import asynccontextmanager @@ -271,15 +272,23 @@ def main() -> None: f"{config.server_host}:{config.server_port}" ) + # Multiple uvicorn workers are safe because assignment, session/revocation + # and receipt authority live in PostgreSQL/Redis, never process memory. + # ``CODE4ME_WEB_WORKERS`` (or ``WEB_CONCURRENCY``) selects the count. + workers = int( + os.environ.get("CODE4ME_WEB_WORKERS") + or os.environ.get("WEB_CONCURRENCY") + or "1" + ) uvicorn.run( "main:app", host=config.server_host, port=config.server_port, log_level="info", access_log=True, + workers=workers if workers > 1 else None, # loop="uvloop", # http="httptools", - # workers=4, ) diff --git a/src/research/__init__.py b/src/research/__init__.py new file mode 100644 index 0000000..89682e3 --- /dev/null +++ b/src/research/__init__.py @@ -0,0 +1,8 @@ +"""Research platform core packages. + +This package is intentionally backend-agnostic: it must never import ``App``, +FastAPI, or a SQLAlchemy session factory. The API layer +(``backend.routers.research``) supplies sockets and persistence; the core +modules here only hold domain models, deterministic serialization, redaction, +receipt building/validation and the compatibility evaluator. +""" diff --git a/src/research/analysis/__init__.py b/src/research/analysis/__init__.py new file mode 100644 index 0000000..b27ac09 --- /dev/null +++ b/src/research/analysis/__init__.py @@ -0,0 +1,11 @@ +"""Researcher analysis: read models and pilot operations. + +Merged home for the scoped researcher read models/RBAC and pilot operations +(health, release gate, kill switch and retention). The individual sub-packages +keep their original public surfaces: + +* :mod:`research.analysis.read_models` +* :mod:`research.analysis.operations` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/analysis/operations/__init__.py b/src/research/analysis/operations/__init__.py new file mode 100644 index 0000000..f79fd0d --- /dev/null +++ b/src/research/analysis/operations/__init__.py @@ -0,0 +1,99 @@ +"""Pilot operations, retention drills, and the release gate (Issue 13). + +Public surface: + +* :mod:`research.analysis.operations.enums` - release/pilot/health/retention vocabularies + and typed reason codes. +* :mod:`research.analysis.operations.models` - ``OperationalHealthV1`` (metadata only), + immutable ``ReleaseEvidenceV1``, ``PilotRunV1``, kill-switch and retention + verification contracts. +* :mod:`research.analysis.operations.health` - pure health aggregation with explicit + ``UNKNOWN`` on a monitoring outage (health is never invented). +* :mod:`research.analysis.operations.release_gate` - ``evaluate_release``. +* :mod:`research.analysis.operations.kill_switch` - auditable scoped kill switch and the + injected predicate used by bootstrap/ingestion. +* :mod:`research.analysis.operations.retention` - deletion-drill verification. +* :mod:`research.analysis.operations.store` - Session-supplied persistence helpers. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from . import store +from .enums import ( + EvidenceState, + HealthSignalState, + KillSwitchScopeKind, + OperationsReasonCode, + PilotClassification, + PilotOutcome, + ReleaseDecision, + RetentionVerificationStatus, +) +from .health import HealthInputs, HealthThresholds, aggregate_health, health_summary +from .kill_switch import ( + KillSwitchRegistry, + engage_kill_switch, + is_engaged, + kill_switch_check, + kill_switch_issue, + release_kill_switch, +) +from .models import ( + ComponentEvidence, + DeleteVerification, + FaultScenario, + HealthSignal, + IncidentRecord, + KillSwitchRecord, + KillSwitchScope, + OperationalHealthV1, + OperationsIssue, + PilotRunV1, + ReleaseEvaluationResult, + ReleaseEvidenceV1, + ReleaseReason, + RetentionVerification, + ReviewerApproval, +) +from .release_gate import evaluate_release, record_decision +from .retention import verify_deletion + +__all__ = [ + "ComponentEvidence", + "DeleteVerification", + "EvidenceState", + "FaultScenario", + "HealthInputs", + "HealthSignal", + "HealthSignalState", + "HealthThresholds", + "IncidentRecord", + "KillSwitchRecord", + "KillSwitchRegistry", + "KillSwitchScope", + "KillSwitchScopeKind", + "OperationalHealthV1", + "OperationsIssue", + "OperationsReasonCode", + "PilotClassification", + "PilotOutcome", + "PilotRunV1", + "ReleaseDecision", + "ReleaseEvaluationResult", + "ReleaseEvidenceV1", + "ReleaseReason", + "RetentionVerification", + "RetentionVerificationStatus", + "ReviewerApproval", + "aggregate_health", + "engage_kill_switch", + "evaluate_release", + "health_summary", + "is_engaged", + "kill_switch_check", + "kill_switch_issue", + "record_decision", + "release_kill_switch", + "store", + "verify_deletion", +] diff --git a/src/research/analysis/operations/enums.py b/src/research/analysis/operations/enums.py new file mode 100644 index 0000000..b06d20e --- /dev/null +++ b/src/research/analysis/operations/enums.py @@ -0,0 +1,118 @@ +"""Closed vocabularies for pilot operations, retention and the release gate. + +Every member is part of a persisted contract (health rows, release evidence, +kill-switch records, retention verifications); members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "EvidenceState", + "HealthSignalState", + "KillSwitchScopeKind", + "OperationsReasonCode", + "PilotClassification", + "PilotOutcome", + "ReleaseDecision", + "RetentionVerificationStatus", +] + + +class ReleaseDecision(str, Enum): + """Terminal outcome of the release gate.""" + + GO = "GO" + NO_GO = "NO_GO" + GO_WITH_LIMITS = "GO_WITH_LIMITS" + EXPIRED = "EXPIRED" + + +class PilotClassification(str, Enum): + """Whether a pilot ran on synthetic or consented data.""" + + SYNTHETIC = "SYNTHETIC" + CONSENTED = "CONSENTED" + + +class HealthSignalState(str, Enum): + """State of one operational health signal.""" + + HEALTHY = "HEALTHY" + DEGRADED = "DEGRADED" + UNKNOWN = "UNKNOWN" + CRITICAL = "CRITICAL" + + +class PilotOutcome(str, Enum): + """Terminal outcome of a pilot run.""" + + COMPLETED = "COMPLETED" + COMPLETED_WITH_INCIDENTS = "COMPLETED_WITH_INCIDENTS" + FAILED = "FAILED" + ABORTED = "ABORTED" + + +class EvidenceState(str, Enum): + """Presence/validity of one required component or evidence item.""" + + PRESENT = "PRESENT" + MISSING = "MISSING" + UNKNOWN = "UNKNOWN" + EXPIRED = "EXPIRED" + CHANGED = "CHANGED" + + +class KillSwitchScopeKind(str, Enum): + """The scope a kill switch applies to.""" + + STUDY = "STUDY" + REVISION = "REVISION" + ENROLLMENT = "ENROLLMENT" + + +class RetentionVerificationStatus(str, Enum): + """Result of a deletion-retention drill.""" + + VERIFIED = "VERIFIED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + + +class OperationsReasonCode(str, Enum): + """Stable machine-readable reason for an operations/gate outcome.""" + + OK = "OK" + + # Health signals. + MONITORING_OUTAGE = "MONITORING_OUTAGE" + MISSING_HEALTH_INPUT = "MISSING_HEALTH_INPUT" + THRESHOLD_DEGRADED = "THRESHOLD_DEGRADED" + THRESHOLD_CRITICAL = "THRESHOLD_CRITICAL" + SPOOL_BACKLOG = "SPOOL_BACKLOG" + UPLOAD_ACK_LOW = "UPLOAD_ACK_LOW" + RUNTIME_EXIT_SPIKE = "RUNTIME_EXIT_SPIKE" + CAPABILITY_MISMATCH = "CAPABILITY_MISMATCH" + RETENTION_INCOMPLETE = "RETENTION_INCOMPLETE" + EXPORT_UNHEALTHY = "EXPORT_UNHEALTHY" + SESSION_ANOMALY = "SESSION_ANOMALY" + + # Release gate. + REQUIRED_COMPONENT_MISSING = "REQUIRED_COMPONENT_MISSING" + REQUIRED_EVIDENCE_MISSING = "REQUIRED_EVIDENCE_MISSING" + COMPONENT_RECEIPT_EXPIRED = "COMPONENT_RECEIPT_EXPIRED" + REVISION_EXPIRED = "REVISION_EXPIRED" + MATERIAL_CHANGE = "MATERIAL_CHANGE" + LIMITATIONS_PRESENT = "LIMITATIONS_PRESENT" + EVIDENCE_INCOMPLETE = "EVIDENCE_INCOMPLETE" + + # Kill switch. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + KILL_SWITCH_RELEASED = "KILL_SWITCH_RELEASED" + + # Retention drill. + RETENTION_VERIFIED = "RETENTION_VERIFIED" + RETENTION_PARTIAL = "RETENTION_PARTIAL" + RETENTION_FAILED = "RETENTION_FAILED" + EXPORT_EXCLUSION_UNVERIFIED = "EXPORT_EXCLUSION_UNVERIFIED" diff --git a/src/research/analysis/operations/health.py b/src/research/analysis/operations/health.py new file mode 100644 index 0000000..f84eff4 --- /dev/null +++ b/src/research/analysis/operations/health.py @@ -0,0 +1,271 @@ +"""Operational health aggregation and alert thresholds (Issue 13). + +Health is computed from injected metadata inputs only. A monitoring outage marks +the affected window ``UNKNOWN`` and never invents health: no metric is inferred +from a missing observation, and missing coverage stays visible. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import HealthSignalState, OperationsReasonCode +from .models import HealthSignal, OperationalHealthV1 + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "HealthInputs", + "HealthThresholds", + "aggregate_health", + "health_summary", +] + +_BASE = ConfigDict(extra="forbid") + +_MONITORING_OUTAGE_REASON = "monitoring outage: health is unknown for this window" + + +class HealthThresholds(BaseModel): + """Threshold configuration for the operational signals.""" + + model_config = _BASE + + spool_depth_degraded: int = 1_000 + spool_depth_critical: int = 10_000 + spool_age_degraded_seconds: int = 3_600 + spool_age_critical_seconds: int = 86_400 + # ACK rate is a ratio; lower is worse. + upload_ack_rate_degraded: float = 0.99 + upload_ack_rate_critical: float = 0.95 + runtime_exits_degraded: int = 1 + runtime_exits_critical: int = 5 + capability_mismatch_degraded: int = 1 + capability_mismatch_critical: int = 5 + + +class HealthInputs(BaseModel): + """Metadata-only health inputs (no content fields).""" + + model_config = _BASE + + spool_depth: Optional[int] = None + spool_age_seconds: Optional[int] = None + upload_ack_rate: Optional[float] = None + proxy_start_failures: Optional[int] = None + runtime_exit_count: Optional[int] = None + capability_mismatch_count: Optional[int] = None + session_state_counts: dict[str, int] = Field(default_factory=dict) + ingestion_reject_reasons: dict[str, int] = Field(default_factory=dict) + retention_job_status: Optional[str] = None + export_job_health: Optional[str] = None + + +def _signal( + name: str, + state: HealthSignalState, + *, + threshold: Optional[str] = None, + observed: Optional[float] = None, + reason: Optional[str] = None, +) -> HealthSignal: + return HealthSignal( + name=name, + state=state, + threshold=threshold, + observed_value=None if observed is None else float(observed), + reason=reason, + ) + + +def _missing(name: str) -> HealthSignal: + return _signal( + name, + HealthSignalState.UNKNOWN, + reason="no observation for this window; health is not inferred", + ) + + +def _upper_signal( + name: str, + value: Optional[int], + *, + degraded: int, + critical: int, +) -> HealthSignal: + if value is None: + return _missing(name) + threshold = f"degraded>={degraded}, critical>={critical}" + if value >= critical: + return _signal(name, HealthSignalState.CRITICAL, threshold=threshold, observed=value) + if value >= degraded: + return _signal(name, HealthSignalState.DEGRADED, threshold=threshold, observed=value) + return _signal(name, HealthSignalState.HEALTHY, threshold=threshold, observed=value) + + +def _ack_rate_signal(value: Optional[float], thresholds: HealthThresholds) -> HealthSignal: + if value is None: + return _missing("upload_ack_rate") + threshold = ( + f"degraded<={thresholds.upload_ack_rate_degraded}, " + f"critical<={thresholds.upload_ack_rate_critical}" + ) + if value <= thresholds.upload_ack_rate_critical: + return _signal( + "upload_ack_rate", HealthSignalState.CRITICAL, threshold=threshold, observed=value + ) + if value <= thresholds.upload_ack_rate_degraded: + return _signal( + "upload_ack_rate", HealthSignalState.DEGRADED, threshold=threshold, observed=value + ) + return _signal( + "upload_ack_rate", HealthSignalState.HEALTHY, threshold=threshold, observed=value + ) + + +def _status_signal(name: str, value: Optional[str]) -> HealthSignal: + if value is None: + return _missing(name) + normalized = value.strip().upper() + if normalized in {"OK", "COMPLETE", "COMPLETED", "HEALTHY"}: + state = HealthSignalState.HEALTHY + elif normalized in {"PENDING", "RUNNING", "DEGRADED", "PARTIAL"}: + state = HealthSignalState.DEGRADED + else: + state = HealthSignalState.CRITICAL + return _signal(name, state, threshold="OK/COMPLETE healthy", observed=None) + + +def aggregate_health( + study_id: UUID, + window_start: datetime, + window_end: datetime, + inputs: HealthInputs, + *, + thresholds: Optional[HealthThresholds] = None, + monitoring_available: bool = True, + now: Optional[datetime] = None, +) -> OperationalHealthV1: + """Build an :class:`OperationalHealthV1` from metadata inputs.""" + config = thresholds or HealthThresholds() + captured = now or datetime.now(timezone.utc) + + if not monitoring_available: + # Never invent health during a monitoring outage. + signals = [ + _signal( + name, + HealthSignalState.UNKNOWN, + threshold=None, + observed=None, + reason=_MONITORING_OUTAGE_REASON, + ) + for name in ( + "spool_depth", + "spool_age_seconds", + "upload_ack_rate", + "runtime_exit_count", + "capability_mismatch_count", + "retention_job_status", + "export_job_health", + ) + ] + return OperationalHealthV1( + study_id=study_id, + window_start=window_start, + window_end=window_end, + session_state_counts={}, + ingestion_reject_reasons={}, + signals=signals, + captured_at=captured, + ) + + signals = [ + _upper_signal( + "spool_depth", + inputs.spool_depth, + degraded=config.spool_depth_degraded, + critical=config.spool_depth_critical, + ), + _upper_signal( + "spool_age_seconds", + inputs.spool_age_seconds, + degraded=config.spool_age_degraded_seconds, + critical=config.spool_age_critical_seconds, + ), + _ack_rate_signal(inputs.upload_ack_rate, config), + _upper_signal( + "runtime_exit_count", + inputs.runtime_exit_count, + degraded=config.runtime_exits_degraded, + critical=config.runtime_exits_critical, + ), + _upper_signal( + "capability_mismatch_count", + inputs.capability_mismatch_count, + degraded=config.capability_mismatch_degraded, + critical=config.capability_mismatch_critical, + ), + _status_signal("retention_job_status", inputs.retention_job_status), + _status_signal("export_job_health", inputs.export_job_health), + ] + return OperationalHealthV1( + study_id=study_id, + window_start=window_start, + window_end=window_end, + spool_depth=inputs.spool_depth, + spool_age_seconds=inputs.spool_age_seconds, + upload_ack_rate=inputs.upload_ack_rate, + proxy_start_failures=inputs.proxy_start_failures, + runtime_exit_count=inputs.runtime_exit_count, + capability_mismatch_count=inputs.capability_mismatch_count, + session_state_counts=dict(inputs.session_state_counts), + ingestion_reject_reasons=dict(inputs.ingestion_reject_reasons), + retention_job_status=inputs.retention_job_status, + export_job_health=inputs.export_job_health, + signals=signals, + captured_at=captured, + ) + + +_SEVERITY = { + HealthSignalState.HEALTHY: 0, + HealthSignalState.UNKNOWN: 1, + HealthSignalState.DEGRADED: 2, + HealthSignalState.CRITICAL: 3, +} + + +def health_summary(snapshot: OperationalHealthV1) -> dict: + """Return the worst-state summary plus explicit per-state counts.""" + overall = HealthSignalState.HEALTHY + counts = {state.value: 0 for state in HealthSignalState} + reasons = [] + for signal in snapshot.signals: + counts[signal.state.value] = counts.get(signal.state.value, 0) + 1 + if _SEVERITY[signal.state] > _SEVERITY[overall]: + overall = signal.state + if signal.state in (HealthSignalState.DEGRADED, HealthSignalState.CRITICAL): + reasons.append( + { + "code": ( + OperationsReasonCode.THRESHOLD_CRITICAL.value + if signal.state == HealthSignalState.CRITICAL + else OperationsReasonCode.THRESHOLD_DEGRADED.value + ), + "signal": signal.name, + "observed_value": signal.observed_value, + "reason": signal.reason, + } + ) + return { + "study_id": str(snapshot.study_id), + "overall": overall.value, + "counts": counts, + "reasons": reasons, + "signals": [signal.model_dump(mode="json") for signal in snapshot.signals], + } diff --git a/src/research/analysis/operations/kill_switch.py b/src/research/analysis/operations/kill_switch.py new file mode 100644 index 0000000..b7c523f --- /dev/null +++ b/src/research/analysis/operations/kill_switch.py @@ -0,0 +1,204 @@ +"""Operator kill switch (Issue 13). + +A kill switch is engaged at a study, revision, or enrollment scope and is +auditable (actor, reason, engaged_at). While engaged it blocks new session +bootstraps and rejects new telemetry batches; those services consult it through +an injected zero-argument predicate so they keep no operations dependency. + +The registry is pure/in-memory; the router can rebuild it from persisted rows. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Callable, Iterable, Optional +from uuid import UUID + +from .enums import OperationsReasonCode +from .models import KillSwitchRecord, KillSwitchScope, OperationsIssue + +__all__ = [ + "KillSwitchRegistry", + "engage_kill_switch", + "is_engaged", + "kill_switch_check", + "kill_switch_issue", + "release_kill_switch", +] + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _matches( + record: KillSwitchRecord, + *, + study_id: Optional[UUID], + revision_id: Optional[UUID], + enrollment_id: Optional[UUID], +) -> bool: + return record.scope.matches( + study_id=study_id, revision_id=revision_id, enrollment_id=enrollment_id + ) + + +class KillSwitchRegistry: + """In-memory collection of kill-switch records.""" + + def __init__(self, records: Optional[Iterable[KillSwitchRecord]] = None) -> None: + self._records: list[KillSwitchRecord] = list(records or []) + + @property + def records(self) -> list[KillSwitchRecord]: + """All records, newest engagement first.""" + return sorted(self._records, key=lambda record: record.engaged_at, reverse=True) + + def add(self, record: KillSwitchRecord) -> KillSwitchRecord: + """Register a record (e.g. rehydrated from storage).""" + self._records.append(record) + return record + + def engage( + self, + scope: KillSwitchScope, + reason: str, + *, + actor: Optional[str] = None, + now: Optional[datetime] = None, + effective_until: Optional[datetime] = None, + ) -> KillSwitchRecord: + """Engage a kill switch at ``scope``.""" + record = KillSwitchRecord( + switch_id=uuid.uuid4(), + scope=scope, + reason=reason, + actor=actor, + engaged_at=_now(now), + effective_until=effective_until, + ) + self._records.append(record) + return record + + def release( + self, switch_id: UUID, *, now: Optional[datetime] = None + ) -> Optional[KillSwitchRecord]: + """Release a switch, returning the updated record or ``None``.""" + for index, record in enumerate(self._records): + if record.switch_id == switch_id: + released = record.model_copy(update={"released_at": _now(now)}) + self._records[index] = released + return released + return None + + def active(self, *, now: Optional[datetime] = None) -> list[KillSwitchRecord]: + """Currently engaged records.""" + timestamp = _now(now) + return [record for record in self.records if record.is_engaged(timestamp)] + + def is_engaged( + self, + *, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, + ) -> bool: + """Whether any active switch covers the requested identifiers.""" + timestamp = _now(now) + return any( + record.is_engaged(timestamp) + and _matches( + record, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + ) + for record in self._records + ) + + +def engage_kill_switch( + registry: KillSwitchRegistry, + scope: KillSwitchScope, + reason: str, + *, + actor: Optional[str] = None, + now: Optional[datetime] = None, + effective_until: Optional[datetime] = None, +) -> KillSwitchRecord: + """Engage a kill switch (auditable scope + reason + actor).""" + return registry.engage( + scope, reason, actor=actor, now=now, effective_until=effective_until + ) + + +def release_kill_switch( + registry: KillSwitchRegistry, switch_id: UUID, *, now: Optional[datetime] = None +) -> Optional[KillSwitchRecord]: + """Release a previously engaged switch.""" + return registry.release(switch_id, now=now) + + +def is_engaged( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> bool: + """Whether a kill switch blocks the requested scope.""" + return registry.is_engaged( + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + now=now, + ) + + +def kill_switch_issue( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> Optional[OperationsIssue]: + """Typed ``KILL_SWITCH_ENGAGED`` issue when the scope is blocked.""" + if is_engaged( + registry, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + now=now, + ): + return OperationsIssue( + code=OperationsReasonCode.KILL_SWITCH_ENGAGED, + message="an operator kill switch is engaged for this scope", + field="kill_switch", + ) + return None + + +def kill_switch_check( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> Callable[[], bool]: + """Return an injected predicate for bootstrap/ingestion.""" + + def check() -> bool: + return is_engaged( + registry, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + now=now, + ) + + return check diff --git a/src/research/analysis/operations/models.py b/src/research/analysis/operations/models.py new file mode 100644 index 0000000..b118b39 --- /dev/null +++ b/src/research/analysis/operations/models.py @@ -0,0 +1,276 @@ +"""Pydantic v2 contracts for pilot operations, retention and the release gate. + +``OperationalHealthV1`` deliberately carries **no content field**: it records +metadata/health signals only (`extra="forbid"` rejects any unknown key such as a +prompt, source, or secret). ``ReleaseEvidenceV1`` is immutable; any unknown or +missing required item yields ``NO_GO``. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.study.protocol.enums import RetentionAction # noqa: TC001 - pydantic field type +from research.telemetry.enums import CoverageState + +from .enums import ( + EvidenceState, + HealthSignalState, + KillSwitchScopeKind, + OperationsReasonCode, + PilotClassification, + PilotOutcome, + ReleaseDecision, + RetentionVerificationStatus, +) + +_BASE = ConfigDict(extra="forbid") +_FROZEN = ConfigDict(extra="forbid", frozen=True) + +__all__ = [ + "ComponentEvidence", + "DeleteVerification", + "FaultScenario", + "HealthSignal", + "IncidentRecord", + "KillSwitchRecord", + "KillSwitchScope", + "OperationalHealthV1", + "OperationsIssue", + "PilotRunV1", + "ReleaseEvaluationResult", + "ReleaseEvidenceV1", + "ReleaseReason", + "RetentionVerification", + "ReviewerApproval", +] + + +class OperationsIssue(BaseModel): + """One typed operations rejection reason.""" + + model_config = _BASE + + code: OperationsReasonCode + message: str + field: str = "" + + +class HealthSignal(BaseModel): + """One evaluated health signal (metadata only).""" + + model_config = _FROZEN + + name: str + state: HealthSignalState = HealthSignalState.UNKNOWN + threshold: Optional[str] = None + observed_value: Optional[float] = None + reason: Optional[str] = None + + +class OperationalHealthV1(BaseModel): + """Operational health snapshot. Contains no content/prompt/secret field.""" + + model_config = _FROZEN + + study_id: UUID + window_start: datetime + window_end: datetime + spool_depth: Optional[int] = None + spool_age_seconds: Optional[int] = None + upload_ack_rate: Optional[float] = None + proxy_start_failures: Optional[int] = None + runtime_exit_count: Optional[int] = None + capability_mismatch_count: Optional[int] = None + session_state_counts: dict[str, int] = Field(default_factory=dict) + ingestion_reject_reasons: dict[str, int] = Field(default_factory=dict) + retention_job_status: Optional[str] = None + export_job_health: Optional[str] = None + signals: list[HealthSignal] = Field(default_factory=list) + captured_at: datetime + + +class ReviewerApproval(BaseModel): + """One reviewer's recorded approval (or rejection).""" + + model_config = _FROZEN + + reviewer_id: str + role: str + approved: bool + approved_at: Optional[datetime] = None + + +class ComponentEvidence(BaseModel): + """Per-component digest/state/expiry used by the gate.""" + + model_config = _FROZEN + + name: str + digest: Optional[str] = None + state: EvidenceState = EvidenceState.PRESENT + expires_at: Optional[datetime] = None + + +class ReleaseEvidenceV1(BaseModel): + """Immutable release evidence consumed by the release gate.""" + + model_config = _FROZEN + + release_id: UUID + study_revision_id: UUID + revision_digest: Optional[str] = None + revision_expires_at: Optional[datetime] = None + # name -> artifact/receipt digest. A blank or UNKNOWN digest is not a pass. + component_artifacts: dict[str, str] = Field(default_factory=dict) + component_expires_at: dict[str, datetime] = Field(default_factory=dict) + # name -> PASS/FAIL/UNKNOWN (anything other than PASS is not a pass). + test_results: dict[str, str] = Field(default_factory=dict) + known_limitations: list[str] = Field(default_factory=list) + reviewer_approvals: list[ReviewerApproval] = Field(default_factory=list) + decision: Optional[ReleaseDecision] = None + recorded_at: Optional[datetime] = None + + +class ReleaseReason(BaseModel): + """One typed gate reason.""" + + model_config = _BASE + + code: OperationsReasonCode + message: str + field: str = "" + + +class ReleaseEvaluationResult(BaseModel): + """Typed release-gate decision with its reasons.""" + + model_config = _BASE + + decision: ReleaseDecision + reasons: list[ReleaseReason] = Field(default_factory=list) + revision_digest: Optional[str] = None + evaluated_at: datetime + evidence: ReleaseEvidenceV1 + + +class FaultScenario(BaseModel): + """One injected fault and its observed outcome.""" + + model_config = _FROZEN + + name: str + injected: bool = True + observed_outcome: str + state: HealthSignalState = HealthSignalState.UNKNOWN + + +class IncidentRecord(BaseModel): + """One pilot incident (metadata only).""" + + model_config = _FROZEN + + incident_id: str + summary: str + severity: str + opened_at: datetime + resolved_at: Optional[datetime] = None + + +class PilotRunV1(BaseModel): + """A recorded synthetic/consented pilot run.""" + + model_config = _FROZEN + + pilot_run_id: UUID + classification: PilotClassification + cohort: list[str] = Field(default_factory=list) + window_start: datetime + window_end: datetime + fault_scenarios: list[FaultScenario] = Field(default_factory=list) + metrics_snapshot_ref: Optional[str] = None + coverage_snapshot_ref: Optional[str] = None + incidents: list[IncidentRecord] = Field(default_factory=list) + outcome: PilotOutcome + recorded_at: datetime + + +class KillSwitchScope(BaseModel): + """The scope a kill switch applies to.""" + + model_config = _FROZEN + + kind: KillSwitchScopeKind + scope_id: UUID + + def matches( + self, + *, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + ) -> bool: + """Whether this scope covers the requested identifiers.""" + if self.kind == KillSwitchScopeKind.STUDY: + return study_id is not None and study_id == self.scope_id + if self.kind == KillSwitchScopeKind.REVISION: + return revision_id is not None and revision_id == self.scope_id + return enrollment_id is not None and enrollment_id == self.scope_id + + +class KillSwitchRecord(BaseModel): + """One auditable kill-switch engagement (or release).""" + + model_config = _BASE + + switch_id: UUID + scope: KillSwitchScope + reason: str + actor: Optional[str] = None + engaged_at: datetime + effective_until: Optional[datetime] = None + released_at: Optional[datetime] = None + + def is_engaged(self, now: datetime) -> bool: + """Whether the switch is currently engaged.""" + if self.released_at is not None: + return False + if self.effective_until is not None and now >= self.effective_until: + return False + return True + + +class DeleteVerification(BaseModel): + """Per-record deletion verification evidence (metadata only).""" + + model_config = _FROZEN + + record_id: str + expected_state: str + observed_state: str + verified: bool + + +class RetentionVerification(BaseModel): + """Result of a deletion-retention drill for one enrollment.""" + + model_config = _BASE + + verification_id: UUID + enrollment_id: UUID + action: RetentionAction + status: RetentionVerificationStatus + evidence_digest: str + verified_at: datetime + deleted_count: int = 0 + remaining_count: int = 0 + expected_remaining_count: int = 0 + export_exclusion_verified: Optional[bool] = None + coverage: CoverageState = CoverageState.UNKNOWN + close_out_allowed: bool = False + reasons: list[OperationsIssue] = Field(default_factory=list) + record_checks: list[DeleteVerification] = Field(default_factory=list) diff --git a/src/research/analysis/operations/release_gate.py b/src/research/analysis/operations/release_gate.py new file mode 100644 index 0000000..4a36bdb --- /dev/null +++ b/src/research/analysis/operations/release_gate.py @@ -0,0 +1,185 @@ +"""Release gate evaluator (Issue 13). + +``evaluate_release`` is a pure function over immutable evidence plus the +required component/evidence lists. Any unknown or missing required item yields +``NO_GO``; an expired component receipt or revision yields ``EXPIRED``; a GO +with recorded limitations is explicitly ``GO_WITH_LIMITS``. A material +artifact/revision change invalidates an old GO. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Iterable, Optional + +from .enums import OperationsReasonCode, ReleaseDecision +from .models import ( + ReleaseEvaluationResult, + ReleaseEvidenceV1, + ReleaseReason, +) + +__all__ = [ + "EXPIRED_REASONS", + "HARD_REASONS", + "NON_BLOCKING_REASONS", + "evaluate_release", + "record_decision", +] + +HARD_REASONS = frozenset( + { + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + OperationsReasonCode.REQUIRED_EVIDENCE_MISSING, + OperationsReasonCode.MATERIAL_CHANGE, + OperationsReasonCode.EVIDENCE_INCOMPLETE, + } +) + +EXPIRED_REASONS = frozenset( + { + OperationsReasonCode.COMPONENT_RECEIPT_EXPIRED, + OperationsReasonCode.REVISION_EXPIRED, + } +) + +NON_BLOCKING_REASONS = frozenset({OperationsReasonCode.LIMITATIONS_PRESENT}) + +_PASS = "PASS" +_UNKNOWN_DIGESTS = frozenset({"", "UNKNOWN", "MISSING", "LATEST"}) + + +def _reason( + code: OperationsReasonCode, message: str, field: str = "" +) -> ReleaseReason: + return ReleaseReason(code=code, message=message, field=field) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def evaluate_release( + evidence: ReleaseEvidenceV1, + *, + required_components: Iterable[str], + required_evidence: Iterable[str], + now: Optional[datetime] = None, + current_component_digests: Optional[dict[str, str]] = None, + current_revision_digest: Optional[str] = None, +) -> ReleaseEvaluationResult: + """Evaluate release evidence and return the typed gate decision.""" + timestamp = _now(now) + reasons: list[ReleaseReason] = [] + + for name in required_components: + digest = evidence.component_artifacts.get(name) + if digest is None: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + f"required component {name!r} has no recorded digest", + name, + ) + ) + elif digest.strip().upper() in _UNKNOWN_DIGESTS: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + f"required component {name!r} digest is unknown", + name, + ) + ) + + expires_at = evidence.component_expires_at.get(name) + if expires_at is not None and expires_at < timestamp: + reasons.append( + _reason( + OperationsReasonCode.COMPONENT_RECEIPT_EXPIRED, + f"component {name!r} receipt expired at {expires_at.isoformat()}", + name, + ) + ) + + if current_component_digests is not None: + current = current_component_digests.get(name) + if current is not None and digest is not None and current != digest: + reasons.append( + _reason( + OperationsReasonCode.MATERIAL_CHANGE, + f"component {name!r} digest changed since the recorded evidence", + name, + ) + ) + + for name in required_evidence: + state = evidence.test_results.get(name) + if state is None or state.strip().upper() != _PASS: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_EVIDENCE_MISSING, + f"required evidence {name!r} is missing or not PASS", + name, + ) + ) + + if evidence.revision_expires_at is not None and evidence.revision_expires_at < timestamp: + reasons.append( + _reason( + OperationsReasonCode.REVISION_EXPIRED, + "the study revision expired at " + f"{evidence.revision_expires_at.isoformat()}", + "study_revision_id", + ) + ) + + if ( + current_revision_digest is not None + and evidence.revision_digest is not None + and current_revision_digest != evidence.revision_digest + ): + reasons.append( + _reason( + OperationsReasonCode.MATERIAL_CHANGE, + "the study revision digest changed since the recorded evidence", + "study_revision_id", + ) + ) + + codes = {reason.code for reason in reasons} + if codes & EXPIRED_REASONS: + decision = ReleaseDecision.EXPIRED + elif codes & HARD_REASONS: + decision = ReleaseDecision.NO_GO + elif evidence.known_limitations: + reasons.append( + _reason( + OperationsReasonCode.LIMITATIONS_PRESENT, + "release is approved with recorded limitations: " + + "; ".join(evidence.known_limitations), + "known_limitations", + ) + ) + decision = ReleaseDecision.GO_WITH_LIMITS + else: + decision = ReleaseDecision.GO + + return ReleaseEvaluationResult( + decision=decision, + reasons=reasons, + revision_digest=evidence.revision_digest, + evaluated_at=timestamp, + evidence=evidence, + ) + + +def record_decision( + evidence: ReleaseEvidenceV1, + decision: ReleaseDecision, + *, + recorded_at: Optional[datetime] = None, +) -> ReleaseEvidenceV1: + """Return an immutable copy of ``evidence`` with the gate decision recorded.""" + return evidence.model_copy( + update={"decision": decision, "recorded_at": recorded_at or _now(None)} + ) diff --git a/src/research/analysis/operations/retention.py b/src/research/analysis/operations/retention.py new file mode 100644 index 0000000..53f8c09 --- /dev/null +++ b/src/research/analysis/operations/retention.py @@ -0,0 +1,180 @@ +"""Deletion-retention drill verification (Issue 13). + +Given an enrollment, its pre-deletion records, the observed post-deletion +records, and (optionally) the exported record ids, verify that the declared +retention action was actually executed and that exports exclude deleted data. + +A partial failure does **not** silently close out: the verification status is +``PARTIAL``/``FAILED``, ``close_out_allowed`` is ``False``, and ledger evidence +(the expected deletion ledger plus an evidence digest) is retained. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional, Sequence +from uuid import UUID + +from research.canonical import canonical_hash +from research.participants import identity as participants_identity +from research.telemetry.enums import CoverageState + +from .enums import OperationsReasonCode, RetentionVerificationStatus +from .models import DeleteVerification, OperationsIssue, RetentionVerification + +if TYPE_CHECKING: + from research.participants.models import Enrollment, PseudonymousRecord + from research.study.protocol.enums import RetentionAction + +__all__ = ["verify_deletion"] + +_LINKABLE = ("account_id", "email", "session_token") + + +def _issue( + code: OperationsReasonCode, message: str, field: str = "" +) -> OperationsIssue: + return OperationsIssue(code=code, message=message, field=field) + + +def _has_linkable(record: PseudonymousRecord) -> bool: + return any(getattr(record, name) is not None for name in _LINKABLE) + + +def verify_deletion( + enrollment: Enrollment, + *, + action: RetentionAction, + before_records: Sequence[PseudonymousRecord], + after_records: Optional[Sequence[PseudonymousRecord]] = None, + export_record_ids: Optional[Sequence[str]] = None, + verified_at: Optional[datetime] = None, + verification_id: Optional[UUID] = None, +) -> RetentionVerification: + """Verify a declared retention action was executed for one enrollment.""" + timestamp = verified_at or datetime.now(timezone.utc) + expected = participants_identity.apply_retention( + enrollment, list(before_records), action, now=timestamp + ) + observed = ( + list(after_records) if after_records is not None else list(expected.records) + ) + + expected_by_id = {record.record_id: record for record in expected.records} + observed_by_id = {record.record_id: record for record in observed} + + before_ids = {record.record_id for record in before_records} + expected_ids = set(expected_by_id) + deleted_ids = before_ids - expected_ids + observed_ids = set(observed_by_id) + + checks: list[DeleteVerification] = [] + missing_present = sorted(expected_ids - observed_ids) + extra_present = sorted(observed_ids - expected_ids) + linkable_left = sorted( + record_id + for record_id, record in observed_by_id.items() + if _has_linkable(record) + ) + + for record_id in sorted(before_ids | observed_ids): + should_exist = record_id in expected_ids + does_exist = record_id in observed_by_id + checks.append( + DeleteVerification( + record_id=record_id, + expected_state="present" if should_exist else "deleted", + observed_state="present" if does_exist else "deleted", + verified=should_exist == does_exist, + ) + ) + + reasons: list[OperationsIssue] = [] + if missing_present: + status = RetentionVerificationStatus.FAILED + reasons.append( + _issue( + OperationsReasonCode.RETENTION_FAILED, + f"{len(missing_present)} retained record(s) were missing after deletion", + "after_records", + ) + ) + elif extra_present or linkable_left: + status = RetentionVerificationStatus.PARTIAL + if extra_present: + reasons.append( + _issue( + OperationsReasonCode.RETENTION_PARTIAL, + f"{len(extra_present)} record(s) were not deleted", + "after_records", + ) + ) + if linkable_left: + reasons.append( + _issue( + OperationsReasonCode.RETENTION_PARTIAL, + f"{len(linkable_left)} record(s) still expose account-linkable fields", + "after_records", + ) + ) + else: + status = RetentionVerificationStatus.VERIFIED + reasons.append( + _issue( + OperationsReasonCode.RETENTION_VERIFIED, + "declared retention action executed for every record", + ) + ) + + export_exclusion_verified: Optional[bool] = None + if export_record_ids is not None: + exported = set(export_record_ids) + export_exclusion_verified = not (deleted_ids & exported) + if not export_exclusion_verified: + status = RetentionVerificationStatus.PARTIAL + reasons.append( + _issue( + OperationsReasonCode.EXPORT_EXCLUSION_UNVERIFIED, + "an export still references deleted record(s)", + "export_record_ids", + ) + ) + + coverage = ( + CoverageState.AVAILABLE + if status == RetentionVerificationStatus.VERIFIED + else CoverageState.PARTIAL + ) + close_out_allowed = ( + status == RetentionVerificationStatus.VERIFIED + and export_exclusion_verified is not False + ) + + evidence_digest = canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment.enrollment_id), + "expected_ledger": expected.ledger.model_dump(mode="json"), + "observed_ids": sorted(observed_ids), + "deleted_ids": sorted(deleted_ids), + "status": status.value, + } + ) + + return RetentionVerification( + verification_id=verification_id or uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + action=action, + status=status, + evidence_digest=evidence_digest, + verified_at=timestamp, + deleted_count=len(deleted_ids), + remaining_count=len(observed_ids), + expected_remaining_count=len(expected_ids), + export_exclusion_verified=export_exclusion_verified, + coverage=coverage, + close_out_allowed=close_out_allowed, + reasons=reasons, + record_checks=checks, + ) diff --git a/src/research/analysis/operations/store.py b/src/research/analysis/operations/store.py new file mode 100644 index 0000000..b3f9b4b --- /dev/null +++ b/src/research/analysis/operations/store.py @@ -0,0 +1,382 @@ +"""Persistence helpers for pilot operations, release gate and kill switch. + +Every helper takes a caller-supplied SQLAlchemy ``Session`` and returns domain +models (never raw ORM rows) so the domain layer stays free of DB/framework +imports. Kill-switch engagements, health snapshots, release evidence and pilot +runs are all consolidated into the single generic ``research_record`` table. + +Deletion-drill *verification* is a pure function +(:func:`research.analysis.operations.retention.verify_deletion`); the single +retention-evidence writer is the identity package's ``insert_deletion_ledger``, +which writes ``research_record`` (``kind = RETENTION_EVIDENCE``) and merges onto +``research_retention_job.evidence_json``. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Callable, Optional, Sequence + +from sqlalchemy import select + +from database.research_schemas import ( + RECORD_KIND_HEALTH, + RECORD_KIND_KILL_SWITCH, + RECORD_KIND_PILOT_RUN, + RECORD_KIND_RELEASE_EVIDENCE, + ResearchRecord, +) +from research.analysis.operations.kill_switch import KillSwitchRegistry + +from .models import ( + KillSwitchRecord, + OperationalHealthV1, + PilotRunV1, + ReleaseEvidenceV1, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from .enums import PilotClassification + +__all__ = [ + "create_health", + "create_pilot_run", + "create_release_evidence", + "db_kill_switch_check", + "engage_kill_switch", + "get_health", + "get_kill_switch", + "get_pilot_run", + "get_release_evidence", + "is_kill_switch_engaged", + "kill_switch_records", + "list_health", + "list_pilot_runs", + "list_release_evidence", + "release_kill_switch", + "row_to_kill_switch", + "set_release_decision", +] + + +def _record( + *, + record_id: uuid.UUID, + kind: str, + occurred_at: datetime, + payload_json: dict, + scope_type: Optional[str] = None, + scope_id: Optional[uuid.UUID] = None, + study_id: Optional[uuid.UUID] = None, + actor: Optional[str] = None, +) -> ResearchRecord: + return ResearchRecord( + record_id=record_id, + kind=kind, + scope_type=scope_type, + scope_id=scope_id, + study_id=study_id, + actor=actor, + occurred_at=occurred_at, + payload_json=payload_json, + ) + + +# --------------------------------------------------------------------------- +# Operational health +# --------------------------------------------------------------------------- + + +def create_health(session: Session, snapshot: OperationalHealthV1) -> ResearchRecord: + """Insert one health snapshot (the full model is stored as JSON).""" + row = _record( + record_id=uuid.uuid4(), + kind=RECORD_KIND_HEALTH, + occurred_at=snapshot.captured_at, + payload_json=snapshot.model_dump(mode="json"), + scope_type="study", + scope_id=snapshot.study_id, + study_id=snapshot.study_id, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def _row_to_health(row: ResearchRecord) -> OperationalHealthV1: + return OperationalHealthV1.model_validate(row.payload_json) + + +def get_health( + session: Session, health_id: uuid.UUID +) -> Optional[OperationalHealthV1]: + """Fetch one health snapshot by id, or ``None``.""" + row = session.get(ResearchRecord, health_id) + if row is None or row.kind != RECORD_KIND_HEALTH: + return None + return _row_to_health(row) + + +def list_health( + session: Session, study_id: uuid.UUID +) -> Sequence[OperationalHealthV1]: + """List a study's health snapshots, oldest-first.""" + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.study_id == study_id, + ResearchRecord.kind == RECORD_KIND_HEALTH, + ) + .order_by(ResearchRecord.occurred_at.asc()) + ) + return [_row_to_health(row) for row in session.execute(statement).scalars().all()] + + +# --------------------------------------------------------------------------- +# Release evidence +# --------------------------------------------------------------------------- + + +def create_release_evidence( + session: Session, evidence: ReleaseEvidenceV1 +) -> ReleaseEvidenceV1: + """Insert one immutable release-evidence record.""" + row = _record( + record_id=evidence.release_id, + kind=RECORD_KIND_RELEASE_EVIDENCE, + occurred_at=evidence.recorded_at or datetime.now(), + payload_json=evidence.model_dump(mode="json"), + scope_type="revision", + scope_id=evidence.study_revision_id, + ) + session.add(row) + session.commit() + session.refresh(row) + return evidence + + +def _row_to_release_evidence(row: ResearchRecord) -> ReleaseEvidenceV1: + return ReleaseEvidenceV1.model_validate(row.payload_json) + + +def get_release_evidence( + session: Session, release_id: uuid.UUID +) -> Optional[ReleaseEvidenceV1]: + """Fetch one release-evidence record by id, or ``None``.""" + row = session.get(ResearchRecord, release_id) + if row is None or row.kind != RECORD_KIND_RELEASE_EVIDENCE: + return None + return _row_to_release_evidence(row) + + +def list_release_evidence( + session: Session, study_revision_id: uuid.UUID +) -> Sequence[ReleaseEvidenceV1]: + """List release evidence for a revision, oldest-first.""" + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.kind == RECORD_KIND_RELEASE_EVIDENCE, + ResearchRecord.payload_json["study_revision_id"].astext + == str(study_revision_id), + ) + .order_by(ResearchRecord.occurred_at.asc()) + ) + return [ + _row_to_release_evidence(row) + for row in session.execute(statement).scalars().all() + ] + + +def set_release_decision( + session: Session, evidence: ReleaseEvidenceV1 +) -> ResearchRecord: + """Record a gate decision on an existing evidence row. + + Only the gate output (the decision fields inside the stored JSON) is + written; the immutable inputs are never changed. + """ + row = session.get(ResearchRecord, evidence.release_id) + if row is None or row.kind != RECORD_KIND_RELEASE_EVIDENCE: + raise ValueError(f"release evidence {evidence.release_id} not found") + row.payload_json = evidence.model_dump(mode="json") + session.add(row) + session.commit() + session.refresh(row) + return row + + +# --------------------------------------------------------------------------- +# Pilot runs +# --------------------------------------------------------------------------- + + +def create_pilot_run(session: Session, run: PilotRunV1) -> PilotRunV1: + """Insert one recorded pilot run.""" + row = _record( + record_id=run.pilot_run_id, + kind=RECORD_KIND_PILOT_RUN, + occurred_at=run.recorded_at, + payload_json=run.model_dump(mode="json"), + ) + session.add(row) + session.commit() + session.refresh(row) + return run + + +def _row_to_pilot_run(row: ResearchRecord) -> PilotRunV1: + return PilotRunV1.model_validate(row.payload_json) + + +def get_pilot_run( + session: Session, pilot_run_id: uuid.UUID +) -> Optional[PilotRunV1]: + """Fetch one pilot run by id, or ``None``.""" + row = session.get(ResearchRecord, pilot_run_id) + if row is None or row.kind != RECORD_KIND_PILOT_RUN: + return None + return _row_to_pilot_run(row) + + +def list_pilot_runs( + session: Session, classification: Optional[PilotClassification] = None +) -> Sequence[PilotRunV1]: + """List pilot runs, optionally filtered by classification.""" + statement = select(ResearchRecord).where( + ResearchRecord.kind == RECORD_KIND_PILOT_RUN + ) + if classification is not None: + statement = statement.where( + ResearchRecord.payload_json["classification"].astext + == classification.value + ) + statement = statement.order_by(ResearchRecord.occurred_at.asc()) + return [ + _row_to_pilot_run(row) + for row in session.execute(statement).scalars().all() + ] + + +# --------------------------------------------------------------------------- +# Kill switch +# --------------------------------------------------------------------------- + + +def engage_kill_switch( + session: Session, record: KillSwitchRecord +) -> KillSwitchRecord: + """Persist a kill-switch engagement.""" + row = _record( + record_id=record.switch_id, + kind=RECORD_KIND_KILL_SWITCH, + occurred_at=record.engaged_at, + payload_json=record.model_dump(mode="json"), + scope_type=record.scope.kind.value, + scope_id=record.scope.scope_id, + actor=record.actor, + ) + session.add(row) + session.commit() + session.refresh(row) + return record + + +def release_kill_switch( + session: Session, switch_id: uuid.UUID, *, + released_at: Optional[datetime] = None, +) -> Optional[KillSwitchRecord]: + """Mark a switch released, returning the record or ``None``.""" + row = session.get(ResearchRecord, switch_id) + if row is None or row.kind != RECORD_KIND_KILL_SWITCH: + return None + payload = dict(row.payload_json or {}) + payload["released_at"] = (released_at or datetime.now()).isoformat() + row.payload_json = payload + session.add(row) + session.commit() + session.refresh(row) + return row_to_kill_switch(row) + + +def get_kill_switch( + session: Session, switch_id: uuid.UUID +) -> Optional[KillSwitchRecord]: + """Fetch a kill-switch record by id, or ``None``.""" + row = session.get(ResearchRecord, switch_id) + if row is None or row.kind != RECORD_KIND_KILL_SWITCH: + return None + return row_to_kill_switch(row) + + +def row_to_kill_switch(row: ResearchRecord) -> KillSwitchRecord: + """Rehydrate a kill-switch record row into a domain record.""" + return KillSwitchRecord.model_validate(row.payload_json) + + +def kill_switch_records(session: Session) -> Sequence[KillSwitchRecord]: + """List all kill-switch records, newest engagement first.""" + statement = ( + select(ResearchRecord) + .where(ResearchRecord.kind == RECORD_KIND_KILL_SWITCH) + .order_by(ResearchRecord.occurred_at.desc()) + ) + return [ + row_to_kill_switch(row) for row in session.execute(statement).scalars().all() + ] + + +def is_kill_switch_engaged( + session: Session, + *, + study_id: Optional[uuid.UUID] = None, + revision_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +) -> bool: + """Whether a persisted ``KILL_SWITCH`` record covers the requested scope. + + This is the DB-backed counterpart of the in-memory + :class:`~research.analysis.operations.kill_switch.KillSwitchRegistry`: it rebuilds the + registry from the persisted records and evaluates the same scope matching + (study/revision/enrollment). A record whose scope kind does not cover the + requested identifiers does not block. + """ + registry = KillSwitchRegistry(kill_switch_records(session)) + return registry.is_engaged( + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + now=now or datetime.now(timezone.utc), + ) + + +def db_kill_switch_check( + session: Session, + *, + study_id: Optional[uuid.UUID] = None, + revision_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +) -> Callable[[], bool]: + """Return a zero-argument kill-switch predicate backed by the database. + + The predicate re-reads the persisted records on every call (the services + invoke it at the point of decision), so an engagement committed between the + request starting and the check taking effect is observed. + """ + + def check() -> bool: + return is_kill_switch_engaged( + session, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + now=now, + ) + + return check diff --git a/src/research/analysis/read_models/__init__.py b/src/research/analysis/read_models/__init__.py new file mode 100644 index 0000000..72d876c --- /dev/null +++ b/src/research/analysis/read_models/__init__.py @@ -0,0 +1,78 @@ +"""Researcher read models and scoped RBAC (Issue 12). + +Read models carry revision id/digest, coverage/derivation version, and the +explicit population definition. Unavailable data is ``null`` + a coverage state, +never zero. + +Public surface: + +* :mod:`research.analysis.read_models.enums` - researcher roles and typed reason codes. +* :mod:`research.analysis.read_models.models` - ``StudyRevisionSummaryV1``, + ``EnrollmentCoverageV1``, ``ConditionExposureV1``, ``TelemetryCoverageV1``, + ``DerivedMetricV1``. +* :mod:`research.analysis.read_models.rbac` - per-study authorization (privacy operator + access is separate from telemetry read). +* :mod:`research.analysis.read_models.service` - pure builders. +* :mod:`research.analysis.read_models.store` - Session-supplied queries. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ReadModelReasonCode, ResearcherRole +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + ConditionExposureV1, + DerivedMetricV1, + EnrollmentCoverageV1, + FamilyCoverage, + ReadModelIssue, + StudyRevisionSummaryV1, + TelemetryCoverageV1, +) +from .rbac import ( + ReadModelAuthorizationError, + ResearcherGrant, + can_read_private_mapping, + can_read_study, + can_read_telemetry, + require_study_access, + require_telemetry_access, + role_for_study, +) +from .service import ( + build_condition_exposures, + build_derived_metrics, + build_enrollment_coverage, + build_revision_summary, + build_telemetry_coverage, + population_definition, +) + +__all__ = [ + "COVERAGE_VERSION", + "DERIVATION_VERSION", + "ConditionExposureV1", + "DerivedMetricV1", + "EnrollmentCoverageV1", + "FamilyCoverage", + "ReadModelAuthorizationError", + "ReadModelIssue", + "ReadModelReasonCode", + "ResearcherGrant", + "ResearcherRole", + "StudyRevisionSummaryV1", + "TelemetryCoverageV1", + "build_condition_exposures", + "build_derived_metrics", + "build_enrollment_coverage", + "build_revision_summary", + "build_telemetry_coverage", + "can_read_private_mapping", + "can_read_study", + "can_read_telemetry", + "population_definition", + "require_study_access", + "require_telemetry_access", + "role_for_study", +] diff --git a/src/research/analysis/read_models/dashboard.py b/src/research/analysis/read_models/dashboard.py new file mode 100644 index 0000000..408749d --- /dev/null +++ b/src/research/analysis/read_models/dashboard.py @@ -0,0 +1,541 @@ +"""Canonical, owner-scoped agent analytics read path (phase 06 cutover). + +One query path serves both the aggregate overview and the per-run detail, reading +the canonical ``research_event`` authority instead of the legacy ``agent_event`` +store. Agent tasks join their canonical events by the explicit phase-05 binding +(``research_event.agent_run_id = agent_task.external_run_id``) — never by an ACP +session id or a time window. + +Ownership is enforced before any join: a non-administrator only ever aggregates +tasks they own (``agent_task.owner_user_id``); an administrator may optionally +scope to one user. A legacy task with no canonical events renders explicit +unavailable/unknown values (``None`` for averages and counts that are genuinely +unknown), never fabricated zeros. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any, Optional + +from sqlalchemy import text + +_TIME_WINDOWS = {"7d": 7, "30d": 30, "90d": 90} + +#: The canonical payload marker the relay/self-report adapters attach to each +#: observation, so a relay ``model_call`` is never confused with an ACP +#: ``agent.message.completed`` for the same run (source ownership). +_LEGACY_KIND = "e.envelope_json -> 'payload' ->> 'legacy_kind'" +_LATENCY = "(e.envelope_json -> 'metrics' ->> 'latency_ms')::float" +_USAGE = "(e.envelope_json -> 'metrics' ->> 'usage_tokens')::bigint" +_UPSTREAM = "e.envelope_json -> 'payload' ->> 'upstream_status'" +_PROMPT_TOKENS = "(e.envelope_json -> 'metrics' -> 'counts' ->> 'prompt_tokens')::bigint" +_COMPLETION_TOKENS = ( + "(e.envelope_json -> 'metrics' -> 'counts' ->> 'completion_tokens')::bigint" +) +_TOOL_NAME = "e.envelope_json -> 'payload' ->> 'tool_name'" +_TOOL_SCHEMA_BYTES = "(e.envelope_json -> 'payload' ->> 'tool_schema_bytes')::bigint" +_CONTEXT_BYTES = ( + "(e.envelope_json -> 'payload' ->> 'context_window_size_bytes')::bigint" +) +_TOOL_RESULT_LENGTH = ( + "(e.envelope_json -> 'payload' ->> 'tool_result_length')::bigint" +) + +_EVENT_JOIN = ( + "FROM research_event e " + "JOIN agent_task t ON t.external_run_id = e.agent_run_id" +) + + +def _seconds_window(time_window: str) -> Optional[int]: + days = _TIME_WINDOWS.get(time_window) + return days + + +def _filters( + current_user, + *, + user_id: Optional[str], + framework: Optional[str], + model: Optional[str], + profile: Optional[str], + start_time: datetime, + end_time: datetime, +) -> tuple[str, dict[str, Any]]: + """Owner scope first, then the optional filters (never before ownership).""" + params: dict[str, Any] = {"start_time": start_time, "end_time": end_time} + conditions = ["t.created_at BETWEEN :start_time AND :end_time"] + is_admin = bool(getattr(current_user, "is_admin", False)) + if is_admin and user_id: + params["user_id"] = user_id + conditions.append("CAST(t.owner_user_id AS TEXT) = :user_id") + elif not is_admin: + params["user_id"] = str(current_user.user_id) + conditions.append("t.owner_user_id = CAST(:user_id AS UUID)") + if framework: + params["framework"] = framework + conditions.append("t.framework_version = :framework") + if model: + params["model"] = model + conditions.append("t.model = :model") + if profile: + params["profile"] = profile + conditions.append("t.agent_profile = :profile") + return " AND ".join(conditions), params + + +def agent_overview( + db: Any, + current_user, + *, + time_window: str, + framework: Optional[str] = None, + model: Optional[str] = None, + profile: Optional[str] = None, + user_id: Optional[str] = None, + now: Optional[datetime] = None, +) -> dict[str, Any]: + """Aggregate canonical agent events for the owner's tasks.""" + days = _seconds_window(time_window) + if days is None: + raise ValueError("time_window must be 7d, 30d, or 90d") + end_time = now or datetime.now(timezone.utc) + start_time = end_time - timedelta(days=days) + where, params = _filters( + current_user, + user_id=user_id, + framework=framework, + model=model, + profile=profile, + start_time=start_time, + end_time=end_time, + ) + + task = db.execute( + text( + f""" + SELECT + COUNT(*) AS total_tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed_tasks, + COUNT(*) FILTER (WHERE t.status IN ('pending', 'running')) AS open_tasks, + COALESCE(AVG(t.total_steps), 0) AS avg_steps, + COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) + AS avg_task_duration_ms + FROM agent_task t + WHERE {where} + """ + ), + params, + ).one() + + events = db.execute( + text( + f""" + SELECT + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS model_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS tool_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS tool_failures, + COUNT(*) FILTER (WHERE e.event_type IN ('agent.error', 'system.proxy.error', + 'system.agent.crashed', 'unknown_source_event') + AND {_LEGACY_KIND} IS NULL) AS observed_failures, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call' + AND ({_UPSTREAM})::int >= 200 AND ({_UPSTREAM})::int < 400) + AS successful_model_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call' + AND ({_UPSTREAM})::int = 429) AS rate_limit_retries, + SUM({_USAGE}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS event_tokens, + SUM({_PROMPT_TOKENS}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS provider_input_tokens, + SUM({_COMPLETION_TOKENS}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS model_output_tokens, + SUM({_TOOL_SCHEMA_BYTES}) AS tool_schema_bytes, + SUM({_CONTEXT_BYTES}) AS conversation_context_bytes, + SUM({_TOOL_RESULT_LENGTH}) AS tool_result_bytes, + AVG({_LATENCY}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS avg_model_latency_ms, + PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY {_LATENCY}) + FILTER (WHERE {_LEGACY_KIND} = 'model_call' AND {_LATENCY} IS NOT NULL) + AS p95_model_latency_ms + {_EVENT_JOIN} + WHERE {where} + """ + ), + params, + ).one() + + profile_rows = db.execute( + text( + f""" + SELECT + COALESCE(t.agent_profile, 'unknown') AS profile_name, + COALESCE(t.framework_version, 'unknown') AS framework_version, + COALESCE(t.model, 'unknown') AS model, + COUNT(*) AS tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE(AVG(t.total_steps), 0) AS avg_steps + FROM agent_task t + WHERE {where} + GROUP BY 1, 2, 3 + ORDER BY tasks DESC, profile_name + """ + ), + params, + ).fetchall() + + tool_rows = db.execute( + text( + f""" + SELECT + COALESCE({_TOOL_NAME}, 'unknown') AS tool_name, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS failures, + COALESCE(AVG({_LATENCY}), 0) AS avg_latency_ms + {_EVENT_JOIN} + WHERE {where} AND {_LEGACY_KIND} IN ('tool_call', 'tool_failed') + GROUP BY 1 + ORDER BY calls DESC, tool_name + LIMIT 20 + """ + ), + params, + ).fetchall() + + run_rows = db.execute( + text( + f""" + SELECT + t.task_id, t.agent_profile, t.framework_version, t.model, t.status, + t.total_steps, t.created_at, t.started_at, t.completed_at, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS model_calls, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS tool_calls, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS failures + FROM agent_task t + LEFT JOIN research_event e ON e.agent_run_id = t.external_run_id + WHERE {where} + GROUP BY t.task_id + ORDER BY t.created_at DESC + LIMIT 25 + """ + ), + params, + ).fetchall() + + trend_rows = db.execute( + text( + f""" + SELECT + DATE_TRUNC('day', t.created_at) AS day, + COUNT(*) AS runs, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) + AS avg_duration_ms + FROM agent_task t + WHERE {where} + GROUP BY 1 + ORDER BY 1 + """ + ), + params, + ).fetchall() + + latency_rows = db.execute( + text( + f""" + SELECT + CASE + WHEN {_LATENCY} < 250 THEN '<250ms' + WHEN {_LATENCY} < 500 THEN '250-499ms' + WHEN {_LATENCY} < 1000 THEN '500-999ms' + WHEN {_LATENCY} < 2000 THEN '1-2s' + ELSE '2s+' + END AS bucket, + CASE + WHEN {_LATENCY} < 250 THEN 1 + WHEN {_LATENCY} < 500 THEN 2 + WHEN {_LATENCY} < 1000 THEN 3 + WHEN {_LATENCY} < 2000 THEN 4 + ELSE 5 + END AS bucket_order, + COUNT(*) AS calls + {_EVENT_JOIN} + WHERE {where} AND {_LEGACY_KIND} = 'model_call' AND {_LATENCY} IS NOT NULL + GROUP BY 1, 2 + ORDER BY bucket_order + """ + ), + params, + ).fetchall() + + event_type_rows = db.execute( + text( + f""" + SELECT + COALESCE({_LEGACY_KIND}, e.event_type) AS event_type, + COUNT(*) AS events + {_EVENT_JOIN} + WHERE {where} + GROUP BY 1 + ORDER BY events DESC, event_type + """ + ), + params, + ).fetchall() + + error_rows = db.execute( + text( + f""" + SELECT + CASE + WHEN {_LEGACY_KIND} = 'tool_failed' THEN 'tool_failed' + WHEN ({_UPSTREAM})::int >= 400 THEN 'upstream_' || ({_UPSTREAM}) + ELSE e.event_type + END AS reason, + COUNT(*) AS events + {_EVENT_JOIN} + WHERE {where} + AND ( + {_LEGACY_KIND} = 'tool_failed' + OR ({_UPSTREAM})::int >= 400 + OR e.event_type IN ('agent.error', 'system.proxy.error', 'system.agent.crashed') + ) + GROUP BY 1 + ORDER BY events DESC, reason + """ + ), + params, + ).fetchall() + + def num(value, integer: bool = False): + if value is None: + return None + return int(value) if integer else float(value) + + total_tasks = num(task.total_tasks, integer=True) or 0 + tool_failures = num(events.tool_failures, integer=True) or 0 + observed_failures = num(events.observed_failures, integer=True) or 0 + return { + "filters": { + "time_window": time_window, + "framework": framework, + "model": model, + "profile": profile, + }, + "summary": { + "total_tasks": total_tasks, + "completed_tasks": num(task.completed_tasks, integer=True) or 0, + "failed_tasks": num(task.failed_tasks, integer=True) or 0, + "open_tasks": num(task.open_tasks, integer=True) or 0, + "completion_rate": (num(task.completed_tasks, integer=True) or 0) + / max(total_tasks, 1), + "avg_steps": num(task.avg_steps), + "avg_task_duration_ms": num(task.avg_task_duration_ms), + "model_calls": num(events.model_calls, integer=True) or 0, + "tool_calls": num(events.tool_calls, integer=True) or 0, + "failures": tool_failures + observed_failures, + # Canonical latencies/tokens are null when no canonical model-call + # fact exists (a legacy run with no canonical events), never zero. + "avg_model_latency_ms": num(events.avg_model_latency_ms), + "p95_model_latency_ms": num(events.p95_model_latency_ms), + "provider_input_tokens": num(events.provider_input_tokens, integer=True), + "model_output_tokens": num(events.model_output_tokens, integer=True), + "event_tokens": num(events.event_tokens, integer=True), + "successful_model_calls": num(events.successful_model_calls, integer=True) or 0, + "rate_limit_retries": num(events.rate_limit_retries, integer=True) or 0, + "tool_schema_tokens_estimated": ( + (num(events.tool_schema_bytes, integer=True) or 0) // 4 + if events.tool_schema_bytes is not None + else None + ), + "conversation_tokens_estimated": ( + (num(events.conversation_context_bytes, integer=True) or 0) // 4 + if events.conversation_context_bytes is not None + else None + ), + "tool_result_tokens_estimated": ( + (num(events.tool_result_bytes, integer=True) or 0) // 4 + if events.tool_result_bytes is not None + else None + ), + # There is no canonical producer for edit acceptance (per the + # contract, it may only be recorded "when truly exposed"); the + # dashboard renders it unavailable rather than computing a ratio. + "edit_acceptance_rate": None, + "total_edits": None, + }, + "profiles": [ + { + "profile_name": row.profile_name, + "framework_version": row.framework_version, + "model": row.model, + "tasks": num(row.tasks, integer=True), + "completed": num(row.completed, integer=True), + "failed": num(row.failed, integer=True), + "completion_rate": (num(row.completed, integer=True) or 0) + / max(num(row.tasks, integer=True) or 0, 1), + "avg_steps": num(row.avg_steps), + } + for row in profile_rows + ], + "tools": [ + { + "tool_name": row.tool_name, + "calls": num(row.calls, integer=True), + "failures": num(row.failures, integer=True), + "avg_latency_ms": num(row.avg_latency_ms), + } + for row in tool_rows + ], + "recent_runs": [ + { + "task_id": str(row.task_id), + "profile_name": row.agent_profile, + "framework_version": row.framework_version, + "model": row.model, + "status": row.status, + "steps": num(row.total_steps, integer=True), + "created_at": _iso(row.created_at), + "started_at": _iso(row.started_at), + "completed_at": _iso(row.completed_at), + "model_calls": num(row.model_calls, integer=True), + "tool_calls": num(row.tool_calls, integer=True), + "failures": num(row.failures, integer=True), + } + for row in run_rows + ], + "trend": [ + { + "day": _iso(row.day), + "runs": num(row.runs, integer=True), + "completed": num(row.completed, integer=True), + "failed": num(row.failed, integer=True), + "avg_duration_ms": num(row.avg_duration_ms), + } + for row in trend_rows + ], + "latency_distribution": [ + {"bucket": row.bucket, "calls": num(row.calls, integer=True)} + for row in latency_rows + ], + "event_types": [ + {"event_type": row.event_type, "events": num(row.events, integer=True)} + for row in event_type_rows + ], + "errors": [ + {"reason": row.reason, "events": num(row.events, integer=True)} + for row in error_rows + ], + } + + +def agent_run_detail( + db: Any, + current_user, + *, + task_id: str, +) -> Optional[dict[str, Any]]: + """Return one owner-scoped run and its canonical events, or ``None``.""" + params: dict[str, Any] = {"task_id": task_id} + owner_filter = "" + if not getattr(current_user, "is_admin", False): + params["user_id"] = str(current_user.user_id) + owner_filter = "AND CAST(t.owner_user_id AS TEXT) = :user_id" + + task = db.execute( + text( + f""" + SELECT t.task_id, t.agent_profile, t.framework_version, t.model, + t.status, t.total_steps, t.created_at, t.started_at, t.completed_at, + t.external_run_id + FROM agent_task t + WHERE CAST(t.task_id AS TEXT) = :task_id {owner_filter} + """ + ), + params, + ).one_or_none() + if task is None: + return None + + event_rows = db.execute( + text( + f""" + SELECT + COALESCE({_LEGACY_KIND}, e.event_type) AS event_type, + e.envelope_json -> 'provenance' ->> 'source' AS source, + {_LATENCY} AS latency_ms, + {_UPSTREAM} AS upstream_status, + {_TOOL_NAME} AS tool_name, + e.envelope_json -> 'payload' ->> 'model' AS model, + {_USAGE} AS total_tokens, + e.occurred_at AS occurred_at, + e.envelope_json -> 'correlations' ->> 'tool_call_id' AS tool_call_id, + e.event_type AS canonical_event_type + FROM research_event e + WHERE e.agent_run_id = :run_id + ORDER BY e.occurred_at, e.emitter_sequence + """ + ), + {"run_id": task.external_run_id}, + ).fetchall() + + events = [] + for position, row in enumerate(event_rows, start=1): + events.append( + { + "event_index": position, + "event_type": row.event_type, + "source": row.source, + "latency_ms": _num(row.latency_ms), + "created_at": None, + "occurred_at": _iso(row.occurred_at), + "span_id": None, + "parent_span_id": None, + "model": row.model, + "tool_name": row.tool_name, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": _num(row.total_tokens), + "finish_reason": None, + "upstream_status": _num(row.upstream_status), + "step_index": None, + "detail": row.canonical_event_type + if row.event_type != row.canonical_event_type + else None, + } + ) + + return { + "run": { + "task_id": str(task.task_id), + "profile_name": task.agent_profile, + "framework_version": task.framework_version, + "model": task.model, + "status": task.status, + "steps": int(task.total_steps or 0), + "created_at": _iso(task.created_at), + "started_at": _iso(task.started_at), + "completed_at": _iso(task.completed_at), + }, + "events": events, + } + + +def _iso(value): + return value.isoformat() if value is not None else None + + +def _num(value): + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/src/research/analysis/read_models/enums.py b/src/research/analysis/read_models/enums.py new file mode 100644 index 0000000..8f268f3 --- /dev/null +++ b/src/research/analysis/read_models/enums.py @@ -0,0 +1,38 @@ +"""Closed vocabularies for researcher read models (Issue 12). + +Roles and reason codes are additive-only contracts: a new role or reason is a +new member, never a renamed one. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["ReadModelReasonCode", "ResearcherRole"] + + +class ResearcherRole(str, Enum): + """Researcher access role granted per study. + + ``PRIVACY_OPERATOR`` is deliberately separate: it may read the private + account-to-participant mapping but **never** telemetry content/read models. + """ + + OWNER = "OWNER" + ANALYST = "ANALYST" + VIEWER = "VIEWER" + PRIVACY_OPERATOR = "PRIVACY_OPERATOR" + + +class ReadModelReasonCode(str, Enum): + """Stable machine-readable reason a read-model request was rejected.""" + + OK = "OK" + FORBIDDEN_STUDY = "FORBIDDEN_STUDY" + TELEMETRY_FORBIDDEN = "TELEMETRY_FORBIDDEN" + NO_ROLE = "NO_ROLE" + NO_REVISION = "NO_REVISION" + REVISION_MISMATCH = "REVISION_MISMATCH" + INVALID_POPULATION = "INVALID_POPULATION" + RETENTION_BLOCKED = "RETENTION_BLOCKED" + WRITER_UNAVAILABLE = "WRITER_UNAVAILABLE" diff --git a/src/research/analysis/read_models/models.py b/src/research/analysis/read_models/models.py new file mode 100644 index 0000000..4a0199e --- /dev/null +++ b/src/research/analysis/read_models/models.py @@ -0,0 +1,141 @@ +"""Pydantic v2 contracts for researcher read models (Issue 12). + +Every read model carries the revision id/digest, the coverage/derivation +version, and the explicit population/filter definition, so a number can never be +read without its provenance. Missing measurement data is represented as +``value: null`` plus an explicit :class:`~research.telemetry.enums.CoverageState`, +never as ``0`` or "complete". +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState +from research.telemetry.projections.models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + FamilyCoverage, +) + +from .enums import ( + ReadModelReasonCode, # noqa: TC001 - pydantic resolves model field annotations at runtime +) + +_BASE = ConfigDict(extra="forbid") + +__all__ = [ + "COVERAGE_VERSION", + "DERIVATION_VERSION", + "ConditionExposureV1", + "DerivedMetricV1", + "EnrollmentCoverageV1", + "FamilyCoverage", + "ReadModelIssue", + "StudyRevisionSummaryV1", + "TelemetryCoverageV1", +] + + +class StudyRevisionSummaryV1(BaseModel): + """An immutable published revision summary for the control plane.""" + + model_config = _BASE + + study_id: UUID + revision_id: UUID + revision_number: int + revision_digest: str + status: str + protocol_schema_version: str + condition_count: int + condition_ids: list[str] = Field(default_factory=list) + published_at: Optional[datetime] = None + supersedes_revision_id: Optional[UUID] = None + created_at: Optional[datetime] = None + population: str = "revision" + + +class EnrollmentCoverageV1(BaseModel): + """Enrollment-state coverage for one revision/population.""" + + model_config = _BASE + + study_id: UUID + revision_id: UUID + revision_digest: str + coverage_version: str = COVERAGE_VERSION + population: str + total_enrollments: int = 0 + status_counts: dict[str, int] = Field(default_factory=dict) + withdrawn_count: int = 0 + coverage: CoverageState = CoverageState.UNKNOWN + coverage_reason: Optional[str] = None + + +class ConditionExposureV1(BaseModel): + """Assignment vs exposure for one condition (separate facts).""" + + model_config = _BASE + + study_id: UUID + revision_id: UUID + revision_digest: str + coverage_version: str = COVERAGE_VERSION + population: str + condition_id: str + assigned_count: int = 0 + exposed_count: int = 0 + non_exposure_count: int = 0 + # None until at least one assignment exists; never a fabricated 0 rate. + exposure_rate: Optional[float] = None + coverage: CoverageState = CoverageState.UNKNOWN + coverage_reason: Optional[str] = None + + +class TelemetryCoverageV1(BaseModel): + """Per-event-family telemetry coverage for one revision/population.""" + + model_config = _BASE + + study_id: UUID + revision_id: UUID + revision_digest: str + coverage_version: str = COVERAGE_VERSION + population: str + families: list[FamilyCoverage] = Field(default_factory=list) + denominators: dict[str, int] = Field(default_factory=dict) + computed_at: Optional[datetime] = None + + +class DerivedMetricV1(BaseModel): + """A versioned derived metric that preserves null/coverage semantics.""" + + model_config = _BASE + + study_id: UUID + revision_id: UUID + revision_digest: str + derivation_version: str = DERIVATION_VERSION + population: str + metric_id: str + numerator: Optional[int] = None + denominator: Optional[int] = None + value: Optional[float] = None + coverage_state: CoverageState = CoverageState.UNKNOWN + coverage_predicate: str + source_event_count: int = 0 + + +class ReadModelIssue(BaseModel): + """One typed read-model rejection reason.""" + + model_config = _BASE + + code: ReadModelReasonCode + message: str + field: str = "" diff --git a/src/research/analysis/read_models/rbac.py b/src/research/analysis/read_models/rbac.py new file mode 100644 index 0000000..44bafdb --- /dev/null +++ b/src/research/analysis/read_models/rbac.py @@ -0,0 +1,128 @@ +"""Scoped researcher RBAC (Issue 12). + +Authorization is per study. A researcher with a role on one study is rejected for +another study: scope is never partially widened by role alone. The privacy +operator capability is separate from telemetry read access. +""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves ResearcherGrant.study_id at runtime + +from pydantic import BaseModel, ConfigDict + +from .enums import ReadModelReasonCode, ResearcherRole + +__all__ = [ + "ReadModelAuthorizationError", + "ResearcherGrant", + "can_read_private_mapping", + "can_read_study", + "can_read_telemetry", + "require_study_access", + "require_telemetry_access", + "role_for_study", +] + +_BASE = ConfigDict(extra="forbid") + +_TELEMETRY_ROLES = frozenset( + {ResearcherRole.OWNER, ResearcherRole.ANALYST, ResearcherRole.VIEWER} +) +_PRIVATE_MAPPING_ROLES = frozenset({ResearcherRole.PRIVACY_OPERATOR}) + + +class ResearcherGrant(BaseModel): + """One researcher's role grant on one study.""" + + model_config = _BASE + + researcher_id: str + study_id: UUID + role: ResearcherRole + + +class ReadModelAuthorizationError(PermissionError): + """Typed authorization failure for a read-model/export request.""" + + def __init__(self, code: ReadModelReasonCode, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def role_for_study( + grants: Iterable[ResearcherGrant], + researcher_id: str, + study_id: UUID, +) -> Optional[ResearcherRole]: + """Return the researcher's role on ``study_id``, or ``None``.""" + for grant in grants: + if grant.researcher_id == researcher_id and grant.study_id == study_id: + return grant.role + return None + + +def can_read_study( + role: Optional[ResearcherRole], + study_id: UUID, + *, + permitted_study_ids: Iterable[UUID], +) -> bool: + """Whether ``role`` may read ``study_id``. + + Requires BOTH an assigned role and an explicit study permission. A role on a + different study never grants access here. + """ + if role is None: + return False + permitted = set(permitted_study_ids) + return study_id in permitted + + +def can_read_telemetry(role: Optional[ResearcherRole]) -> bool: + """Whether ``role`` may read telemetry read models/exports. + + Privacy operators are excluded: they work with the private mapping, never + telemetry content. + """ + return role in _TELEMETRY_ROLES + + +def can_read_private_mapping(role: Optional[ResearcherRole]) -> bool: + """Whether ``role`` may read the private account-to-participant mapping.""" + return role in _PRIVATE_MAPPING_ROLES + + +def require_study_access( + grants: Iterable[ResearcherGrant], + researcher_id: str, + study_id: UUID, +) -> ResearcherRole: + """Return the role or raise a typed authorization error.""" + grants = list(grants) + role = role_for_study(grants, researcher_id, study_id) + permitted = [grant.study_id for grant in grants if grant.researcher_id == researcher_id] + if not can_read_study(role, study_id, permitted_study_ids=permitted): + # Distinguish "this researcher has no grants at all" from "this + # researcher has grants elsewhere but not on this study". + code = ( + ReadModelReasonCode.NO_ROLE + if not permitted + else ReadModelReasonCode.FORBIDDEN_STUDY + ) + raise ReadModelAuthorizationError( + code, f"researcher is not authorized for study {study_id}" + ) + assert role is not None + return role + + +def require_telemetry_access(role: ResearcherRole) -> None: + """Raise when ``role`` may not read telemetry read models.""" + if not can_read_telemetry(role): + raise ReadModelAuthorizationError( + ReadModelReasonCode.TELEMETRY_FORBIDDEN, + f"role {role.value} may not read telemetry read models", + ) diff --git a/src/research/analysis/read_models/service.py b/src/research/analysis/read_models/service.py new file mode 100644 index 0000000..27769c3 --- /dev/null +++ b/src/research/analysis/read_models/service.py @@ -0,0 +1,244 @@ +"""Pure read-model builders (Issue 12). + +Each builder is deterministic over injected canonical records / assignments / +exposures / enrollments and carries the revision id+digest, the coverage or +derivation version, and the explicit population definition. Missing data stays +visible: ``value``/``exposure_rate`` are ``None`` with an explicit coverage +state, never ``0``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Sequence + +from research.telemetry.enums import CoverageState +from research.telemetry.projections.models import COVERAGE_VERSION +from research.telemetry.projections.service import ( + coverage_by_family, + derive_event_count_metric, + derive_usage_metric, +) + +if TYPE_CHECKING: + from uuid import UUID + +from .models import ( + ConditionExposureV1, + DerivedMetricV1, + EnrollmentCoverageV1, + StudyRevisionSummaryV1, + TelemetryCoverageV1, +) + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from research.runtime.assignment.models import AssignmentV1, ExposureV1 + from research.study.protocol.publication import StudyRevision + +__all__ = [ + "build_condition_exposures", + "build_derived_metrics", + "build_enrollment_coverage", + "build_revision_summary", + "build_telemetry_coverage", + "population_definition", +] + +_POPULATIONS = { + "revision": "all records bound to this revision", + "all_events": "all stored canonical events for the revision", + "enrolled": "all enrolled participants for the revision", + "exposed": "participants with an actual condition exposure", +} + + +def population_definition(population: str) -> str: + """Return the human-readable definition of a population filter.""" + return _POPULATIONS.get(population, f"custom population {population!r}") + + +def _status_value(value: Any) -> str: + return value.value if hasattr(value, "value") else str(value) + + +def build_revision_summary(revision: StudyRevision) -> StudyRevisionSummaryV1: + """Build the immutable revision summary read model.""" + protocol_json = revision.protocol_json or {} + conditions = protocol_json.get("conditions") or [] + condition_ids = [str(condition.get("condition_id", "")) for condition in conditions] + return StudyRevisionSummaryV1( + study_id=revision.study_id, + revision_id=revision.revision_id, + revision_number=revision.revision_number, + revision_digest=revision.protocol_digest, + status=_status_value(revision.status), + protocol_schema_version=str(protocol_json.get("schema_version", "1")), + condition_count=len(conditions), + condition_ids=condition_ids, + published_at=revision.published_at, + supersedes_revision_id=revision.supersedes_revision_id, + created_at=revision.created_at, + population="revision", + ) + + +def build_enrollment_coverage( + enrollments: Iterable[Enrollment], + *, + study_id: UUID, + revision_id: UUID, + revision_digest: str, + population: str = "enrolled", +) -> EnrollmentCoverageV1: + """Summarize enrollment states with explicit coverage.""" + status_counts: dict[str, int] = {} + total = 0 + for enrollment in enrollments: + total += 1 + status = _status_value(enrollment.status) + status_counts[status] = status_counts.get(status, 0) + 1 + if total == 0: + coverage = CoverageState.UNAVAILABLE + reason = "no enrollments in population" + else: + coverage = CoverageState.AVAILABLE + reason = None + return EnrollmentCoverageV1( + study_id=study_id, + revision_id=revision_id, + revision_digest=revision_digest, + coverage_version=COVERAGE_VERSION, + population=population, + total_enrollments=total, + status_counts=status_counts, + withdrawn_count=status_counts.get("WITHDRAWN", 0), + coverage=coverage, + coverage_reason=reason, + ) + + +def build_condition_exposures( + assignments: Iterable[AssignmentV1], + exposures: Iterable[ExposureV1], + *, + study_id: UUID, + revision_id: UUID, + revision_digest: str, + population: str = "revision", +) -> list[ConditionExposureV1]: + """Build per-condition assignment vs exposure read models.""" + assignment_list = list(assignments) + exposures_by_assignment: dict[UUID, list[ExposureV1]] = {} + for exposure in exposures: + exposures_by_assignment.setdefault(exposure.assignment_id, []).append(exposure) + + condition_ids = sorted({assignment.condition_id for assignment in assignment_list}) + results: list[ConditionExposureV1] = [] + for condition_id in condition_ids: + condition_assignments = [ + assignment + for assignment in assignment_list + if assignment.condition_id == condition_id + ] + exposed = 0 + non_exposed = 0 + for assignment in condition_assignments: + outcomes = exposures_by_assignment.get(assignment.assignment_id, []) + if any(exposure.is_exposure for exposure in outcomes): + exposed += 1 + elif outcomes: + non_exposed += 1 + assigned = len(condition_assignments) + results.append( + ConditionExposureV1( + study_id=study_id, + revision_id=revision_id, + revision_digest=revision_digest, + coverage_version=COVERAGE_VERSION, + population=population, + condition_id=condition_id, + assigned_count=assigned, + exposed_count=exposed, + non_exposure_count=non_exposed, + exposure_rate=(exposed / assigned) if assigned else None, + coverage=( + CoverageState.AVAILABLE if assigned else CoverageState.UNAVAILABLE + ), + coverage_reason=None if assigned else "no assignments for condition", + ) + ) + return results + + +def build_telemetry_coverage( + records: Iterable[Any], + *, + study_id: UUID, + revision_id: UUID, + revision_digest: str, + population: str = "all_events", +) -> TelemetryCoverageV1: + """Wrap the shared coverage projection with revision context.""" + projection = coverage_by_family( + list(records), revision_id, population=population + ) + return TelemetryCoverageV1( + study_id=study_id, + revision_id=revision_id, + revision_digest=revision_digest, + coverage_version=projection.coverage_version, + population=projection.population, + families=projection.families, + denominators=projection.denominators, + computed_at=projection.computed_at, + ) + + +def _wrap_metric( + metric: Any, + *, + study_id: UUID, + revision_id: UUID, + revision_digest: str, + population: str, +) -> DerivedMetricV1: + return DerivedMetricV1( + study_id=study_id, + revision_id=revision_id, + revision_digest=revision_digest, + derivation_version=metric.derivation_version, + population=population, + metric_id=metric.metric_id, + numerator=metric.numerator, + denominator=metric.denominator, + value=metric.value, + coverage_state=metric.coverage_state, + coverage_predicate=metric.coverage_predicate, + source_event_count=metric.source_event_count, + ) + + +def build_derived_metrics( + records: Sequence[Any], + *, + study_id: UUID, + revision_id: UUID, + revision_digest: str, + population: str = "all_events", +) -> list[DerivedMetricV1]: + """Build the versioned derived metrics with explicit coverage.""" + records = list(records) + metrics = [ + derive_event_count_metric(records, population=population), + derive_usage_metric(records, population=population), + ] + return [ + _wrap_metric( + metric, + study_id=study_id, + revision_id=revision_id, + revision_digest=revision_digest, + population=population, + ) + for metric in metrics + ] diff --git a/src/research/analysis/read_models/store.py b/src/research/analysis/read_models/store.py new file mode 100644 index 0000000..6d6e616 --- /dev/null +++ b/src/research/analysis/read_models/store.py @@ -0,0 +1,144 @@ +"""Session-supplied read queries for the researcher control plane (Issue 12). + +These helpers are thin wrappers over the canonical stores: they return domain +records (never raw ORM/JSON) and never join login identity into a read model. +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from research.participants.models import Enrollment + from research.runtime.assignment.models import AssignmentV1, ExposureV1 + from research.runtime.sessions.models import AgentRunV1, ResearchSessionV1 + from research.telemetry.ingestion.models import ResearchEventRecord + +from database.research_schemas import ( + ResearchAgentRun, + ResearchEnrollment, + ResearchEvent, + StudyAssignment, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.participants import identity as identity_store +from research.runtime.assignment import store as assignment_store +from research.runtime.sessions import store as session_store +from research.telemetry.ingestion import store as ingestion_store + +__all__ = [ + "list_agent_runs", + "list_assignments", + "list_enrollments", + "list_events", + "list_exposures", + "list_revisions", + "list_sessions", +] + + +def list_revisions( + session: Session, study_id: uuid.UUID +) -> Sequence[object]: + """Return no rows; study revisions are no longer a research authority.""" + return [] + + +def list_enrollments(session: Session, study_id: uuid.UUID) -> Sequence[Enrollment]: + """List a study's enrollments as domain records (no login identity).""" + statement = select(ResearchEnrollment).where(ResearchEnrollment.study_id == study_id) + return [ + identity_store.row_to_enrollment(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_assignments(session: Session, study_id: uuid.UUID) -> Sequence[AssignmentV1]: + """List a study's assignments as domain records (joined via enrollment).""" + statement = ( + select(StudyAssignment) + .join( + ResearchEnrollment, + StudyAssignment.enrollment_id == ResearchEnrollment.enrollment_id, + ) + .where(ResearchEnrollment.study_id == study_id) + ) + return [ + assignment_store.row_to_assignment(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_exposures( + session: Session, revision_id: uuid.UUID +) -> Sequence[ExposureV1]: + """Return no rows; condition exposure persistence was removed.""" + return [] + + +def list_sessions( + session: Session, study_id: uuid.UUID +) -> Sequence[ResearchSessionV1]: + """List a study's research sessions as domain records.""" + statement = ( + select(ResearchSessionRow) + .join(ResearchEnrollment, ResearchSessionRow.enrollment_id == ResearchEnrollment.enrollment_id) + .where(ResearchEnrollment.study_id == study_id) + .order_by(ResearchSessionRow.created_at.asc()) + ) + return [ + session_store.row_to_session(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_agent_runs( + session: Session, research_session_id: uuid.UUID +) -> Sequence[AgentRunV1]: + """List the agent runs of one session as domain records.""" + statement = ( + select(ResearchAgentRun) + .where(ResearchAgentRun.research_session_id == research_session_id) + .order_by(ResearchAgentRun.started_at.asc()) + ) + return [ + session_store.row_to_agent_run(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_events( + session: Session, + revision_id: uuid.UUID, + *, + research_session_ids: Optional[Sequence[uuid.UUID]] = None, +) -> Sequence[ResearchEventRecord]: + """List a revision's canonical events as records (sorted deterministically). + + Retention tombstones (``retention_state == "DELETED"``) are excluded so a + post-withdrawal export never reads deleted data. + """ + statement = select(ResearchEvent).where( + ResearchEvent.revision_id == revision_id, + ResearchEvent.retention_state != "DELETED", + ) + if research_session_ids is not None: + statement = statement.where( + ResearchEvent.research_session_id.in_(list(research_session_ids)) + ) + statement = statement.order_by( + ResearchEvent.research_session_id.asc(), + ResearchEvent.emitter_id.asc(), + ResearchEvent.emitter_sequence.asc(), + ) + return [ + ingestion_store.row_to_record(row) + for row in session.execute(statement).scalars().all() + ] diff --git a/src/research/canonical.py b/src/research/canonical.py new file mode 100644 index 0000000..69684ca --- /dev/null +++ b/src/research/canonical.py @@ -0,0 +1,55 @@ +"""Deterministic canonical JSON and SHA-256 hashing (shared). + +Equivalent documents must hash identically regardless of dictionary insertion +order or non-semantic whitespace. We achieve that by always serializing with +``sort_keys=True`` and compact separators over the *parsed* structure, so an +input string with incidental whitespace collapses to the same canonical bytes. + +This module holds the backend-agnostic primitives shared by every research +artifact (Issue 01 receipts, Issue 02 study protocols, and later issues). The +compatibility package re-exports these names so existing imports keep working. +""" + +from __future__ import annotations + +import hashlib +import json +import uuid +from datetime import date, datetime +from enum import Enum +from typing import Any + + +def _json_default(value: Any) -> Any: + """Coerce common non-JSON types to a stable representation.""" + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Enum): + return value.value + if hasattr(value, "model_dump"): + # Pydantic models nested inside plain dicts. + return value.model_dump(mode="json") + raise TypeError(f"Object of type {type(value).__name__} is not canonical-JSON serializable") + + +def canonical_json(obj: Any) -> str: + """Return the canonical JSON string for ``obj`` (sorted keys, compact).""" + return json.dumps( + obj, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=_json_default, + ) + + +def canonical_bytes(obj: Any) -> bytes: + """UTF-8 encoded :func:`canonical_json`.""" + return canonical_json(obj).encode("utf-8") + + +def canonical_hash(obj: Any) -> str: + """SHA-256 hex digest of :func:`canonical_bytes`.""" + return hashlib.sha256(canonical_bytes(obj)).hexdigest() diff --git a/src/research/compatibility/__init__.py b/src/research/compatibility/__init__.py new file mode 100644 index 0000000..b58daa4 --- /dev/null +++ b/src/research/compatibility/__init__.py @@ -0,0 +1,20 @@ +"""ACP host-evidence and compatibility gate (Issue 01). + +Public surface: + +* :mod:`research.compatibility.enums` - closed vocabularies for capabilities, + states, fidelity, enforcement owners, decisions and reason codes. +* :mod:`research.compatibility.models` - Pydantic v2 contracts for the + ``AcpCapabilityReceiptV1`` (and its versioned sub-objects). +* :mod:`research.compatibility.canonical` - deterministic canonical JSON and + SHA-256 hashing. +* :mod:`research.compatibility.redaction` - recursive secret removal. +* :mod:`research.compatibility.receipt` - fixture-transcript ingestion and + receipt validation. +* :mod:`research.compatibility.evaluate` - the deterministic compatibility + evaluator. + +The persistence adapters live in :mod:`research.compatibility.store` and take +a caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" diff --git a/src/research/compatibility/canonical.py b/src/research/compatibility/canonical.py new file mode 100644 index 0000000..33aaba9 --- /dev/null +++ b/src/research/compatibility/canonical.py @@ -0,0 +1,41 @@ +"""Deterministic canonical JSON and SHA-256 hashing (Issue 01 surface). + +The generic primitives live in :mod:`research.canonical` so every research +artifact (receipts, study protocols, ...) shares one serializer. This module +re-exports them under their historical names and keeps the receipt-specific +``content_hash`` helper that excludes a receipt's own hash field. +""" + +from __future__ import annotations + +from typing import Any + +from research.canonical import ( # noqa: F401 - intentional re-export + canonical_bytes, + canonical_hash, + canonical_json, +) + +# ``content_hash`` is the only field excluded from its own digest. Keeping the +# name in one place avoids a silent drift between the builder and the verifier. +CONTENT_HASH_FIELD = "content_hash" + + +def _dump_without_hash(receipt: Any) -> dict[str, Any]: + if hasattr(receipt, "model_dump"): + data = receipt.model_dump(mode="json") + elif isinstance(receipt, dict): + data = dict(receipt) + else: + raise TypeError("receipt must be a Pydantic model or a mapping") + data.pop(CONTENT_HASH_FIELD, None) + return data + + +def receipt_content_hash(receipt_without_hash: Any) -> str: + """Hash a receipt (model or mapping) ignoring its own ``content_hash``. + + The argument may either already have ``content_hash`` set (in which case it + is stripped before hashing) or be a bare dict/model in its pre-hash form. + """ + return canonical_hash(_dump_without_hash(receipt_without_hash)) diff --git a/src/research/compatibility/enums.py b/src/research/compatibility/enums.py new file mode 100644 index 0000000..9676087 --- /dev/null +++ b/src/research/compatibility/enums.py @@ -0,0 +1,99 @@ +"""Closed vocabularies for the ACP compatibility gate. + +Every value here is part of a persisted contract (receipt JSON, decision +responses, reason codes), so members are additive only: renaming or removing a +member is a breaking change to already-stored receipts. +""" + +from __future__ import annotations + +from enum import Enum + + +class CapabilityId(str, Enum): + """Host/agent capability observed by a pinned fixture probe.""" + + INITIALIZE = "INITIALIZE" + TOOL_LIFECYCLE = "TOOL_LIFECYCLE" + TOOL_UPDATES = "TOOL_UPDATES" + PERMISSION_REQUEST = "PERMISSION_REQUEST" + EDIT_PROPOSAL = "EDIT_PROPOSAL" + DIFF = "DIFF" + CANCELLATION = "CANCELLATION" + SESSION_CLOSE = "SESSION_CLOSE" + SESSION_LOAD = "SESSION_LOAD" + PLANS = "PLANS" + USAGE = "USAGE" + + +class CapabilityState(str, Enum): + """Observed support state for one capability. + + ``UNKNOWN``, ``UNAVAILABLE``, ``PARTIAL`` and ``BROKEN`` are deliberately + distinct from ``SUPPORTED``: an ambiguous or absent observation must never + be promoted to a supported claim. + """ + + SUPPORTED = "SUPPORTED" + PARTIAL = "PARTIAL" + UNKNOWN = "UNKNOWN" + UNAVAILABLE = "UNAVAILABLE" + BROKEN = "BROKEN" + + +class Fidelity(str, Enum): + """How faithfully the observation captures the underlying behavior.""" + + EXACT = "EXACT" + NORMALIZED = "NORMALIZED" + INFERRED = "INFERRED" + + +class EnforcementOwner(str, Enum): + """Which component actually enforces the observed capability.""" + + HOST = "HOST" + AGENT = "AGENT" + PROXY = "PROXY" + PLUGIN = "PLUGIN" + UNKNOWN = "UNKNOWN" + + +class CompatibilityDecision(str, Enum): + """Terminal outcome of evaluating required capabilities.""" + + COMPATIBLE = "COMPATIBLE" + INCOMPATIBLE_ENVIRONMENT = "INCOMPATIBLE_ENVIRONMENT" + INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE" + BROKEN = "BROKEN" + + +class CompatibilityReasonCode(str, Enum): + """Machine-readable justification for a compatibility decision.""" + + OK = "OK" + + # Receipt presence / integrity. + MISSING_RECEIPT = "MISSING_RECEIPT" + RECEIPT_UNVERIFIED = "RECEIPT_UNVERIFIED" + RECEIPT_HASH_MISMATCH = "RECEIPT_HASH_MISMATCH" + + # Environment / identity mismatches. + ENVIRONMENT_MISMATCH = "ENVIRONMENT_MISMATCH" + IDE_BUILD_MISMATCH = "IDE_BUILD_MISMATCH" + AI_ASSISTANT_BUILD_MISMATCH = "AI_ASSISTANT_BUILD_MISMATCH" + PLUGIN_VERSION_MISMATCH = "PLUGIN_VERSION_MISMATCH" + AGENT_ID_MISMATCH = "AGENT_ID_MISMATCH" + AGENT_VERSION_MISMATCH = "AGENT_VERSION_MISMATCH" + ADAPTER_VERSION_MISMATCH = "ADAPTER_VERSION_MISMATCH" + PROTOCOL_VERSION_MISMATCH = "PROTOCOL_VERSION_MISMATCH" + + # Capability evidence gaps. + CAPABILITY_MISSING = "CAPABILITY_MISSING" + CAPABILITY_UNKNOWN = "CAPABILITY_UNKNOWN" + CAPABILITY_PARTIAL = "CAPABILITY_PARTIAL" + CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE" + CAPABILITY_BROKEN = "CAPABILITY_BROKEN" + + # Fixture parsing. + PARSE_FAILED = "PARSE_FAILED" diff --git a/src/research/compatibility/evaluate.py b/src/research/compatibility/evaluate.py new file mode 100644 index 0000000..3fba1f5 --- /dev/null +++ b/src/research/compatibility/evaluate.py @@ -0,0 +1,325 @@ +"""Deterministic compatibility evaluator. + +Every rule below is explicit and order-independent except for the aggregate +severity ranking (``BROKEN`` > ``INCOMPATIBLE_ENVIRONMENT`` > +``INSUFFICIENT_EVIDENCE`` > ``COMPATIBLE``). Evaluation never silently passes a +missing, partial, or unverified requirement. +""" + +from __future__ import annotations + +from typing import Optional + +from .canonical import receipt_content_hash +from .enums import ( + CapabilityState, + CompatibilityDecision, + CompatibilityReasonCode, +) +from .models import ( + AcpCapabilityReceiptV1, + AgentIdentity, + CapabilityEvaluation, + CompatibilityRequest, + CompatibilityResult, + EnvironmentTuple, + ReasonDetail, + RequiredCapability, +) +from .receipt import validate_receipt + +_ENVIRONMENT_FIELD_REASONS = { + "ide_build": CompatibilityReasonCode.IDE_BUILD_MISMATCH, + "ai_assistant_build": CompatibilityReasonCode.AI_ASSISTANT_BUILD_MISMATCH, + "plugin_version": CompatibilityReasonCode.PLUGIN_VERSION_MISMATCH, + "os": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + "arch": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + "host_kind": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, +} + +_AGENT_FIELD_REASONS = { + "agent_id": CompatibilityReasonCode.AGENT_ID_MISMATCH, + "agent_version": CompatibilityReasonCode.AGENT_VERSION_MISMATCH, + "adapter_version": CompatibilityReasonCode.ADAPTER_VERSION_MISMATCH, +} + +# Aggregate severity ranking (higher wins). +_SEVERITY = { + CompatibilityDecision.COMPATIBLE: 0, + CompatibilityDecision.INSUFFICIENT_EVIDENCE: 1, + CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT: 2, + CompatibilityDecision.BROKEN: 3, +} + +_REASON_SEVERITY = { + CompatibilityReasonCode.OK: CompatibilityDecision.COMPATIBLE, + CompatibilityReasonCode.MISSING_RECEIPT: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.RECEIPT_UNVERIFIED: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.RECEIPT_HASH_MISMATCH: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.ENVIRONMENT_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.IDE_BUILD_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AI_ASSISTANT_BUILD_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.PLUGIN_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AGENT_ID_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AGENT_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.ADAPTER_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.CAPABILITY_MISSING: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_UNKNOWN: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_PARTIAL: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_UNAVAILABLE: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.CAPABILITY_BROKEN: CompatibilityDecision.BROKEN, + CompatibilityReasonCode.PARSE_FAILED: CompatibilityDecision.BROKEN, +} + + +def _result( + reasons: list[ReasonDetail], + per_capability: Optional[list[CapabilityEvaluation]] = None, +) -> CompatibilityResult: + if not reasons: + return CompatibilityResult( + decision=CompatibilityDecision.COMPATIBLE, + reasons=[ReasonDetail(code=CompatibilityReasonCode.OK, message="all requirements satisfied")], + per_capability=per_capability or [], + ) + decision = CompatibilityDecision.COMPATIBLE + for reason in reasons: + candidate = _REASON_SEVERITY.get( + reason.code, CompatibilityDecision.INSUFFICIENT_EVIDENCE + ) + if _SEVERITY[candidate] > _SEVERITY[decision]: + decision = candidate + return CompatibilityResult( + decision=decision, + reasons=reasons, + per_capability=per_capability or [], + ) + + +def _environment_reasons( + expected: Optional[EnvironmentTuple], actual: EnvironmentTuple +) -> list[ReasonDetail]: + if expected is None: + return [] + reasons: list[ReasonDetail] = [] + for field, code in _ENVIRONMENT_FIELD_REASONS.items(): + expected_value = getattr(expected, field, None) + if expected_value is None: + continue + actual_value = getattr(actual, field, None) + if actual_value != expected_value: + reasons.append( + ReasonDetail( + code=code, + message=( + f"environment.{field} expected {expected_value!r} " + f"but receipt has {actual_value!r}" + ), + ) + ) + return reasons + + +def _agent_reasons( + expected: Optional[AgentIdentity], actual: AgentIdentity +) -> list[ReasonDetail]: + if expected is None: + return [] + reasons: list[ReasonDetail] = [] + for field, code in _AGENT_FIELD_REASONS.items(): + expected_value = getattr(expected, field, None) + if expected_value is None: + continue + actual_value = getattr(actual, field, None) + if actual_value != expected_value: + reasons.append( + ReasonDetail( + code=code, + message=( + f"agent.{field} expected {expected_value!r} " + f"but receipt has {actual_value!r}" + ), + ) + ) + return reasons + + +def _evaluate_requirement( + receipt: AcpCapabilityReceiptV1, requirement: RequiredCapability +) -> CapabilityEvaluation: + observed = next( + ( + operation + for operation in receipt.observed_operations + if operation.capability == requirement.capability + ), + None, + ) + base = CapabilityEvaluation( + capability=requirement.capability, + required_state=requirement.require_state, + observed_state=observed.state if observed else None, + enforcement_owner=observed.enforcement_owner if observed else None, + satisfied=False, + ) + + if observed is None: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_MISSING, + capability=requirement.capability, + message=( + f"receipt has no observed operation for {requirement.capability.value}" + ), + ) + return base + + state = observed.state + if state == CapabilityState.BROKEN: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_BROKEN, + capability=requirement.capability, + message=f"{requirement.capability.value} is broken in this environment", + ) + return base + + if state == CapabilityState.UNAVAILABLE: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNAVAILABLE, + capability=requirement.capability, + message=f"{requirement.capability.value} is unavailable in this environment", + ) + return base + + if state == CapabilityState.UNKNOWN: + if requirement.require_state == CapabilityState.UNKNOWN: + base.satisfied = True + return base + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNKNOWN, + capability=requirement.capability, + message=f"{requirement.capability.value} was never observed (UNKNOWN)", + ) + return base + + if state == CapabilityState.PARTIAL: + if requirement.require_state in ( + CapabilityState.PARTIAL, + CapabilityState.UNKNOWN, + ): + base.satisfied = True + return base + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_PARTIAL, + capability=requirement.capability, + message=( + f"{requirement.capability.value} is only PARTIAL but " + f"{requirement.require_state.value} was required" + ), + ) + return base + + # Remaining state: SUPPORTED. + if requirement.require_state == CapabilityState.SUPPORTED: + base.satisfied = True + return base + + # require_state is PARTIAL or UNKNOWN, both weaker than SUPPORTED. + base.satisfied = True + return base + + +def evaluate_compatibility(request: CompatibilityRequest) -> CompatibilityResult: + """Evaluate ``request`` and return a typed, justified decision.""" + receipt = request.receipt + if receipt is None: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.MISSING_RECEIPT, + message="no capability receipt was supplied", + ) + ] + ) + + if not isinstance(receipt, AcpCapabilityReceiptV1): + try: + receipt = AcpCapabilityReceiptV1.model_validate(receipt) + except Exception as error: # noqa: BLE001 - report any parse failure + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_UNVERIFIED, + message=f"receipt could not be parsed: {error}", + ) + ] + ) + + validation_reasons = validate_receipt(receipt) + if validation_reasons: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_UNVERIFIED, + capability=reason.capability, + message=f"receipt is not scoped: {reason.message}", + ) + for reason in validation_reasons + ] + ) + + broken = receipt.status == CapabilityState.BROKEN or any( + operation.state == CapabilityState.BROKEN + for operation in receipt.observed_operations + ) + if broken: + if receipt.status == CapabilityState.BROKEN and not receipt.observed_operations: + reason = ReasonDetail( + code=CompatibilityReasonCode.PARSE_FAILED, + message="fixture transcript could not be parsed", + ) + else: + reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_BROKEN, + message="receipt contains a broken capability", + ) + return _result([reason]) + + if receipt_content_hash(receipt) != receipt.content_hash: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_HASH_MISMATCH, + message="receipt content hash does not match its canonical content", + ) + ] + ) + + reasons: list[ReasonDetail] = [] + reasons.extend(_environment_reasons(request.expected_environment, receipt.environment)) + reasons.extend(_agent_reasons(request.expected_agent, receipt.agent)) + if ( + request.expected_protocol_version is not None + and request.expected_protocol_version != receipt.protocol_version + ): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH, + message=( + f"protocol_version expected {request.expected_protocol_version!r} " + f"but receipt has {receipt.protocol_version!r}" + ), + ) + ) + + per_capability = [ + _evaluate_requirement(receipt, requirement) + for requirement in request.requirements + ] + reasons.extend( + evaluation.reason + for evaluation in per_capability + if evaluation.reason is not None + ) + + return _result(reasons, per_capability) diff --git a/src/research/compatibility/models.py b/src/research/compatibility/models.py new file mode 100644 index 0000000..f9c71d1 --- /dev/null +++ b/src/research/compatibility/models.py @@ -0,0 +1,169 @@ +"""Pydantic v2 contracts for the ACP capability receipt and evaluation API. + +These models are the wire + persistence contract for Issue 01. They are +deliberately permissive about *missing* optional evidence (so an incomplete +receipt can be represented and then explicitly rejected by +:func:`research.compatibility.receipt.validate_receipt`) but strict about +enum-typed values. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + CapabilityId, + CapabilityState, + CompatibilityDecision, + CompatibilityReasonCode, + EnforcementOwner, + Fidelity, +) + + +class EnvironmentTuple(BaseModel): + """Version-pinned host environment a receipt was captured against. + + Fields are optional so the same type can express a *partial* expectation in + :class:`CompatibilityRequest` (only present fields are compared). On a + receipt, :func:`research.compatibility.receipt.validate_receipt` requires + the core fields to be present. + """ + + model_config = ConfigDict(extra="ignore") + + ide_build: Optional[str] = None + ai_assistant_build: Optional[str] = None + plugin_version: Optional[str] = None + os: Optional[str] = None + arch: Optional[str] = None + host_kind: Optional[str] = None + + +class AgentIdentity(BaseModel): + """Agent + adapter identity a receipt was captured against. + + ``agent_id`` / ``agent_version`` are required on a receipt (enforced by + validation); they are optional here so a partial expected identity can be + supplied as an override. + """ + + model_config = ConfigDict(extra="ignore") + + agent_id: Optional[str] = None + agent_version: Optional[str] = None + adapter_version: Optional[str] = None + release_digest: Optional[str] = None + executable_digest: Optional[str] = None + + +class EvidenceRef(BaseModel): + """A content-addressed pointer to redacted evidence. + + The raw payload is never stored here; only its digest and a stable + reference are. ``redacted`` records that the pointed-to artifact has been + through :func:`research.compatibility.redaction.redact`. + """ + + model_config = ConfigDict(extra="ignore") + + ref: str + kind: str + sha256: str + size: Optional[int] = None + redacted: bool = True + + +class CapabilityResult(BaseModel): + """Observed result for one capability in one environment.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + state: CapabilityState + fidelity: Fidelity = Fidelity.NORMALIZED + enforcement_owner: EnforcementOwner = EnforcementOwner.UNKNOWN + limitations: list[str] = Field(default_factory=list) + observed_at: Optional[datetime] = None + evidence_refs: list[str] = Field(default_factory=list) + + +class AcpCapabilityReceiptV1(BaseModel): + """Immutable, redacted, hashed capability receipt (schema version 1).""" + + model_config = ConfigDict(extra="ignore") + + schema_version: str = "1" + receipt_id: UUID + captured_at: datetime + environment: EnvironmentTuple + agent: AgentIdentity + protocol_version: str + declared: dict[str, str | bool] = Field(default_factory=dict) + observed_operations: list[CapabilityResult] = Field(default_factory=list) + evidence: list[EvidenceRef] = Field(default_factory=list) + status: CapabilityState + # Content hash over the canonical serialization of this model with + # ``content_hash`` excluded. Empty until finalised by the builder. + content_hash: str = "" + + +class RequiredCapability(BaseModel): + """One capability predicate a caller requires of a receipt.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + require_state: CapabilityState = CapabilityState.SUPPORTED + enforcement_owner: Optional[EnforcementOwner] = None + + +class CompatibilityRequest(BaseModel): + """Evaluate ``requirements`` against a receipt (inline or by id).""" + + model_config = ConfigDict(extra="ignore") + + receipt: Optional[AcpCapabilityReceiptV1] = None + receipt_id: Optional[UUID] = None + requirements: list[RequiredCapability] = Field(default_factory=list) + expected_environment: Optional[EnvironmentTuple] = None + expected_agent: Optional[AgentIdentity] = None + # Negotiated ACP wire protocol the caller requires. + expected_protocol_version: Optional[str] = None + + +class ReasonDetail(BaseModel): + """Typed explanation for one part of a compatibility decision.""" + + model_config = ConfigDict(extra="ignore") + + code: CompatibilityReasonCode + capability: Optional[CapabilityId] = None + message: str + + +class CapabilityEvaluation(BaseModel): + """Per-requirement outcome, independent of the aggregate decision.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + required_state: CapabilityState + observed_state: Optional[CapabilityState] = None + enforcement_owner: Optional[EnforcementOwner] = None + satisfied: bool + reason: Optional[ReasonDetail] = None + + +class CompatibilityResult(BaseModel): + """Aggregate compatibility decision and its full justification.""" + + model_config = ConfigDict(extra="ignore") + + decision: CompatibilityDecision + reasons: list[ReasonDetail] = Field(default_factory=list) + per_capability: list[CapabilityEvaluation] = Field(default_factory=list) diff --git a/src/research/compatibility/receipt.py b/src/research/compatibility/receipt.py new file mode 100644 index 0000000..a0dafb7 --- /dev/null +++ b/src/research/compatibility/receipt.py @@ -0,0 +1,444 @@ +"""Fixture-transcript ingestion and receipt validation. + +The ingestion path turns a captured ACP observation transcript into a redacted, +hashed :class:`~research.compatibility.models.AcpCapabilityReceiptV1`. Raw +payloads are never embedded in the receipt: only redacted digests are, so a +receipt remains safe to persist and export. + +A malformed transcript never fails open. It yields a ``BROKEN`` receipt and a +separate ``acp_parse_failed`` system event built by +:func:`parse_failure_event`. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping, Optional +from uuid import uuid4 + +from pydantic import BaseModel + +from .canonical import canonical_bytes, canonical_hash, receipt_content_hash +from .enums import ( + CapabilityId, + CapabilityState, + CompatibilityReasonCode, + EnforcementOwner, + Fidelity, +) +from .models import ( + AcpCapabilityReceiptV1, + AgentIdentity, + CapabilityResult, + EnvironmentTuple, + EvidenceRef, + ReasonDetail, +) +from .redaction import redact + +# Event contract for a failed fixture parse. Kept as a plain dict because it is +# emitted on the existing ``agent_event`` wire shape (no raw content). +ACP_PARSE_FAILED_EVENT_TYPE = "system.acp_parse_failed" + +_REQUIRED_ENVIRONMENT_FIELDS = ( + "ide_build", + "ai_assistant_build", + "plugin_version", + "os", + "arch", +) + + +def _coerce_fidelity(value: Any) -> Fidelity: + if value is None: + return Fidelity.NORMALIZED + try: + return Fidelity(str(value).strip().upper()) + except ValueError: + return Fidelity.INFERRED + + +def _coerce_owner(value: Any) -> EnforcementOwner: + if value is None: + return EnforcementOwner.UNKNOWN + try: + return EnforcementOwner(str(value).strip().upper()) + except ValueError: + return EnforcementOwner.UNKNOWN + + +def _parse_datetime(value: Any) -> Optional[datetime]: + if value is None or isinstance(value, datetime): + return value + if isinstance(value, str): + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + return datetime.fromisoformat(text) + except ValueError: + return None + return None + + +def _extract_capabilities(message: Any) -> Optional[Mapping[str, Any]]: + """Pull the capability object out of an initialize request/response.""" + if not isinstance(message, Mapping): + return None + for container_key in ("result", "params"): + container = message.get(container_key) + if isinstance(container, Mapping): + for caps_key in ("agentCapabilities", "clientCapabilities"): + caps = container.get(caps_key) + if isinstance(caps, Mapping): + return caps + for caps_key in ("agentCapabilities", "clientCapabilities"): + caps = message.get(caps_key) + if isinstance(caps, Mapping): + return caps + return None + + +def _flatten_capabilities( + prefix: str, value: Any, out: dict[str, str | bool] +) -> None: + if not isinstance(value, Mapping): + return + for key, item in value.items(): + name = f"{prefix}.{key}" if prefix else str(key) + if isinstance(item, Mapping): + _flatten_capabilities(name, item, out) + elif isinstance(item, bool): + out[name] = item + elif isinstance(item, str): + out[name] = item + elif item is None: + out[name] = False + else: + out[name] = str(item) + + +def _declared_capabilities(transcript: Mapping[str, Any]) -> dict[str, str | bool]: + declared: dict[str, str | bool] = {} + raw = transcript.get("declared") + if isinstance(raw, Mapping): + _flatten_capabilities("", raw, declared) + _flatten_capabilities( + "client", _extract_capabilities(transcript.get("initialize_request")), declared + ) + _flatten_capabilities( + "agent", _extract_capabilities(transcript.get("initialize_response")), declared + ) + return declared + + +def _protocol_version(transcript: Mapping[str, Any]) -> str: + direct = transcript.get("protocol_version") + if isinstance(direct, str) and direct.strip(): + return direct.strip() + for message_key in ("initialize_response", "initialize_request"): + message = transcript.get(message_key) + if not isinstance(message, Mapping): + continue + for container_key in ("result", "params"): + container = message.get(container_key) + if isinstance(container, Mapping): + version = container.get("protocolVersion") + if isinstance(version, str) and version.strip(): + return version.strip() + return "unknown" + + +def _evidence_ref_for_operation( + raw: Mapping[str, Any], + capability: CapabilityId, + index: int, + capture_channel: str, +) -> EvidenceRef: + payload = raw.get("evidence") + if payload is None: + payload = {key: raw.get(key) for key in ("method", "id", "result", "error")} + redacted = redact(payload).value + encoded = canonical_bytes(redacted) + return EvidenceRef( + ref=f"fixture://{capture_channel}/operation/{index}/{capability.value.lower()}", + kind="acp_operation", + sha256=canonical_hash(redacted), + size=len(encoded), + redacted=True, + ) + + +def _redacted_transcript_evidence( + transcript: Any, capture_channel: str +) -> EvidenceRef: + redacted = redact(transcript).value + encoded = canonical_bytes(redacted) + return EvidenceRef( + ref=f"fixture://{capture_channel}/transcript.redacted.json", + kind="redacted_transcript", + sha256=canonical_hash(redacted), + size=len(encoded), + redacted=True, + ) + + +def _parse_operations( + transcript: Any, capture_channel: str +) -> tuple[list[CapabilityResult], list[EvidenceRef], Optional[str]]: + """Parse fixture operations, returning ``(results, evidence, parse_error)``.""" + if not isinstance(transcript, Mapping): + return [], [], "fixture transcript is not an object" + + explicit_error = transcript.get("parse_error") + if explicit_error: + return [], [], str(explicit_error) + + raw_operations = transcript.get("operations") + if raw_operations is None: + raw_operations = [] + if not isinstance(raw_operations, list): + return [], [], "fixture 'operations' must be a list" + + results: list[CapabilityResult] = [] + evidence: list[EvidenceRef] = [] + for index, raw in enumerate(raw_operations): + if not isinstance(raw, Mapping): + return [], [], f"operation {index} is not an object" + capability_raw = raw.get("capability") + state_raw = raw.get("state") + if not capability_raw or not state_raw: + return [], [], f"operation {index} is missing capability or state" + try: + capability = CapabilityId(str(capability_raw).strip().upper()) + except ValueError: + return [], [], f"operation {index} has unknown capability {capability_raw!r}" + try: + state = CapabilityState(str(state_raw).strip().upper()) + except ValueError: + return [], [], f"operation {index} has unknown state {state_raw!r}" + + ref = _evidence_ref_for_operation(raw, capability, index, capture_channel) + evidence.append(ref) + + limitations = raw.get("limitations") or [] + if isinstance(limitations, str): + limitations = [limitations] + elif not isinstance(limitations, list): + limitations = [str(limitations)] + + results.append( + CapabilityResult( + capability=capability, + state=state, + fidelity=_coerce_fidelity(raw.get("fidelity")), + enforcement_owner=_coerce_owner(raw.get("enforcement_owner")), + limitations=[str(item) for item in limitations], + observed_at=_parse_datetime(raw.get("observed_at")), + evidence_refs=[ref.ref], + ) + ) + return results, evidence, None + + +def _aggregate_status(results: list[CapabilityResult]) -> CapabilityState: + if not results: + return CapabilityState.UNKNOWN + states = {result.state for result in results} + for state in ( + CapabilityState.BROKEN, + CapabilityState.UNAVAILABLE, + CapabilityState.UNKNOWN, + CapabilityState.PARTIAL, + ): + if state in states: + return state + return CapabilityState.SUPPORTED + + +def build_receipt_from_fixture( + transcript: dict, + environment: EnvironmentTuple, + agent: AgentIdentity, + *, + capture_channel: str, + captured_at: Optional[datetime] = None, +) -> AcpCapabilityReceiptV1: + """Build a redacted, hashed receipt from a captured fixture transcript. + + A transcript is treated as broken when it carries a ``parse_error`` or is + structurally malformed (unknown capability/state, non-object operations, + or a non-object transcript). In that case the receipt has + ``status=BROKEN``, no observed operations, and only the redacted + transcript digest as evidence. + """ + captured = captured_at or datetime.now(timezone.utc) + operations, operation_evidence, parse_error = _parse_operations( + transcript, capture_channel + ) + + if parse_error is not None: + operations = [] + operation_evidence = [] + + evidence = [_redacted_transcript_evidence(transcript, capture_channel)] + evidence.extend(operation_evidence) + + if parse_error is not None: + status = CapabilityState.BROKEN + else: + status = _aggregate_status(operations) + + declared = _declared_capabilities(transcript) if isinstance(transcript, Mapping) else {} + + receipt = AcpCapabilityReceiptV1( + schema_version="1", + receipt_id=uuid4(), + captured_at=captured, + environment=environment, + agent=agent, + protocol_version=( + _protocol_version(transcript) if isinstance(transcript, Mapping) else "unknown" + ), + declared=declared, + observed_operations=operations, + evidence=evidence, + status=status, + content_hash="", + ) + receipt.content_hash = receipt_content_hash(receipt) + return receipt + + +def receipt_parse_error(transcript: Any) -> Optional[str]: + """Return the parse-error reason a transcript would yield, or ``None``.""" + return _parse_operations(transcript, "fixture")[2] + + +def parse_failure_event( + reason: str, + *, + transcript: Any = None, + capture_channel: str = "fixture", + environment: Optional[EnvironmentTuple] = None, + agent: Optional[AgentIdentity] = None, + captured_at: Optional[datetime] = None, +) -> dict: + """Build the ``system.acp_parse_failed`` event for a broken transcript. + + The event carries only a hashed evidence reference and caller-supplied + metadata; it never fabricates probe content. + """ + captured = captured_at or datetime.now(timezone.utc) + evidence = _redacted_transcript_evidence(transcript, capture_channel) + return { + "event_type": ACP_PARSE_FAILED_EVENT_TYPE, + "source": "acp", + "severity": "error", + "reason": str(reason), + "captured_at": captured.isoformat(), + "environment": environment.model_dump(mode="json") if environment else None, + "agent": agent.model_dump(mode="json") if agent else None, + "evidence": evidence.model_dump(mode="json"), + } + + +def _as_mapping(receipt: AcpCapabilityReceiptV1 | Mapping[str, Any]) -> Mapping[str, Any]: + if isinstance(receipt, BaseModel): + return receipt.model_dump(mode="json") + if isinstance(receipt, Mapping): + return receipt + return {} + + +def _safe_capability(value: Any) -> Optional[CapabilityId]: + if value is None: + return None + try: + return CapabilityId(str(value).strip().upper()) + except ValueError: + return None + + +def validate_receipt( + receipt: AcpCapabilityReceiptV1 | Mapping[str, Any], +) -> list[ReasonDetail]: + """Return the reasons a receipt makes an unscoped capability claim. + + An empty list means the receipt is scoped: it names its environment, + agent identity, negotiated protocol version, and every observed operation + carries both a capability and a state. + """ + data = _as_mapping(receipt) + reasons: list[ReasonDetail] = [] + + environment = data.get("environment") + if not isinstance(environment, Mapping): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + message="receipt environment is missing", + ) + ) + else: + for field in _REQUIRED_ENVIRONMENT_FIELDS: + if not environment.get(field): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + message=f"receipt environment.{field} is missing", + ) + ) + + agent = data.get("agent") + if not isinstance(agent, Mapping) or not agent.get("agent_id"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.AGENT_ID_MISMATCH, + message="receipt agent.agent_id is missing", + ) + ) + if not isinstance(agent, Mapping) or not agent.get("agent_version"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.AGENT_VERSION_MISMATCH, + message="receipt agent.agent_version is missing", + ) + ) + + if not data.get("protocol_version"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH, + message="receipt protocol_version is missing", + ) + ) + + operations = data.get("observed_operations") + if operations is None: + operations = [] + if not isinstance(operations, list): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PARSE_FAILED, + message="receipt observed_operations is not a list", + ) + ) + else: + for index, operation in enumerate(operations): + if not isinstance(operation, Mapping) or not operation.get("capability"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_MISSING, + message=f"observed operation {index} is missing a capability", + ) + ) + elif not operation.get("state"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNKNOWN, + capability=_safe_capability(operation.get("capability")), + message=f"observed operation {index} is missing a state", + ) + ) + return reasons diff --git a/src/research/compatibility/redaction.py b/src/research/compatibility/redaction.py new file mode 100644 index 0000000..c995a04 --- /dev/null +++ b/src/research/compatibility/redaction.py @@ -0,0 +1,126 @@ +"""Recursive, non-mutating secret redaction. + +Two layers of defence: + +1. **Key-based** - any mapping key whose lower-cased name contains a known + secret marker (``password``, ``token``, ``api_key`` ...) has its whole value + replaced by ``[REDACTED]``. +2. **Value-based** - string scalars are scanned for well-known inline secret + shapes (``Bearer ``, ``sk-...``, ``ghp_...``, ``AKIA...``) even when + they appear under an innocuous key. + +The input is never mutated: only fresh dicts/lists/strings are returned. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Mapping + +REDACTED = "[REDACTED]" + +# Case-insensitive substrings that mark a key as secret-bearing. +SECRET_KEY_PATTERNS: tuple[str, ...] = ( + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "bearer", + "credential", + "private_key", + "access_key", + "client_secret", + "cookie", + "session_token", +) + +# Inline value shapes. The bearer case preserves the scheme for readability. +_VALUE_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{4,}"), "Bearer " + REDACTED), + (re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"), REDACTED), + (re.compile(r"\bghp_[A-Za-z0-9]{8,}"), REDACTED), + (re.compile(r"\bAKIA[0-9A-Z]{12,}"), REDACTED), +) + + +@dataclass +class RedactionResult: + """A redacted copy plus counts of what was removed. + + ``value`` is a structurally independent copy; the original object passed to + :func:`redact` is untouched. + """ + + value: Any + key_redactions: int = 0 + value_redactions: int = 0 + paths: list[str] = field(default_factory=list) + + @property + def redaction_count(self) -> int: + return self.key_redactions + self.value_redactions + + @property + def changed(self) -> bool: + return self.redaction_count > 0 + + def to_dict(self) -> dict[str, Any]: + return { + "key_redactions": self.key_redactions, + "value_redactions": self.value_redactions, + "redaction_count": self.redaction_count, + "paths": list(self.paths), + } + + +def is_secret_key(key: Any) -> bool: + """True when a mapping key name marks its value as secret material.""" + lowered = str(key).lower() + return any(pattern in lowered for pattern in SECRET_KEY_PATTERNS) + + +def redact_string(value: str) -> tuple[str, int]: + """Return ``(redacted, count)`` for inline secret shapes in one string.""" + result = value + count = 0 + for pattern, replacement in _VALUE_PATTERNS: + result, substitutions = pattern.subn(replacement, result) + count += substitutions + return result, count + + +def redact(value: Any) -> RedactionResult: + """Recursively redact ``value`` and return a :class:`RedactionResult`.""" + result = RedactionResult(value=None) + result.value = _walk(value, result, path="$") + return result + + +def _walk(value: Any, result: RedactionResult, path: str) -> Any: + if isinstance(value, Mapping): + redacted: dict[Any, Any] = {} + for key, item in value.items(): + child_path = f"{path}.{key}" + if is_secret_key(key): + redacted[key] = REDACTED + result.key_redactions += 1 + result.paths.append(child_path) + else: + redacted[key] = _walk(item, result, child_path) + return redacted + if isinstance(value, (list, tuple)): + return [ + _walk(item, result, f"{path}[{index}]") + for index, item in enumerate(value) + ] + if isinstance(value, str): + redacted_value, count = redact_string(value) + if count: + result.value_redactions += count + result.paths.append(path) + return redacted_value + return value diff --git a/src/research/compatibility/store.py b/src/research/compatibility/store.py new file mode 100644 index 0000000..aa0d958 --- /dev/null +++ b/src/research/compatibility/store.py @@ -0,0 +1,111 @@ +"""CRUD-style persistence helpers for capability receipts. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core +package never imports ``App`` or touches the application singleton. The router +is responsible for session lifecycle (``App.get_db_session`` / +``rollback`` / ``close``). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + +from database.research_schemas import AcpCapabilityReceipt + +from .models import AcpCapabilityReceiptV1 + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def persist_receipt(session: Session, receipt: AcpCapabilityReceiptV1) -> AcpCapabilityReceipt: + """Insert a receipt. + + The evidence digests are part of ``receipt_json.evidence[]`` (the canonical + content), so no child rows are written. Raises + ``sqlalchemy.exc.IntegrityError`` if the ``content_hash`` already exists; the + caller decides whether that is a conflict or an idempotent retry. + """ + environment = receipt.environment + agent = receipt.agent + row = AcpCapabilityReceipt( + receipt_id=receipt.receipt_id, + captured_at=receipt.captured_at, + ide_build=environment.ide_build, + ai_assistant_build=environment.ai_assistant_build, + plugin_version=environment.plugin_version, + os=environment.os, + arch=environment.arch, + agent_id=agent.agent_id, + agent_version=agent.agent_version, + adapter_version=agent.adapter_version, + protocol_version=receipt.protocol_version, + status=receipt.status.value, + content_hash=receipt.content_hash, + receipt_json=receipt.model_dump(mode="json"), + created_at=_now(), + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_receipt(session: Session, receipt_id: uuid.UUID) -> Optional[AcpCapabilityReceipt]: + """Fetch a receipt row by primary key, or ``None``.""" + return session.get(AcpCapabilityReceipt, receipt_id) + + +def list_receipts( + session: Session, *, limit: int = 100, offset: int = 0 +) -> Sequence[AcpCapabilityReceipt]: + """List receipts newest-first.""" + statement = ( + select(AcpCapabilityReceipt) + .order_by(AcpCapabilityReceipt.captured_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_receipt(row: AcpCapabilityReceipt) -> AcpCapabilityReceiptV1: + """Rehydrate the stored receipt JSON into the versioned model.""" + return AcpCapabilityReceiptV1.model_validate(row.receipt_json) + + +def row_to_summary(row: AcpCapabilityReceipt) -> dict[str, Any]: + """Return a compact, non-secret summary safe for list responses.""" + captured_at = row.captured_at + created_at = row.created_at + return { + "receipt_id": str(row.receipt_id), + "captured_at": captured_at.isoformat() if captured_at else None, + "created_at": ( + created_at.isoformat() if isinstance(created_at, datetime) else None + ), + "status": row.status, + "content_hash": row.content_hash, + "protocol_version": row.protocol_version, + "environment": { + "ide_build": row.ide_build, + "ai_assistant_build": row.ai_assistant_build, + "plugin_version": row.plugin_version, + "os": row.os, + "arch": row.arch, + }, + "agent": { + "agent_id": row.agent_id, + "agent_version": row.agent_version, + "adapter_version": row.adapter_version, + }, + } diff --git a/src/research/participants/__init__.py b/src/research/participants/__init__.py new file mode 100644 index 0000000..6cc920c --- /dev/null +++ b/src/research/participants/__init__.py @@ -0,0 +1,14 @@ +"""Participant identity, consent, withdrawal and retention. + +Flat home for the participant identity/consent state machine and the +idempotent retention execution service: + +* :mod:`identity` - the pure enrollment/consent/re-consent/withdrawal state + machine, participant-code generation and retention, the researcher-safe + projection, and the SQLAlchemy persistence adapters. +* :mod:`retention` - pure retention planning/execution, the persistence + helpers plus :class:`SqlAlchemyRetentionStore`, and the bounded retrying + worker entry points. + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/participants/enums.py b/src/research/participants/enums.py new file mode 100644 index 0000000..0d1c143 --- /dev/null +++ b/src/research/participants/enums.py @@ -0,0 +1,111 @@ +"""Closed vocabularies for participant identity and retention. + +Consent is a single acceptance recorded once at join (no document identity, no +re-consent, no withdrawal), so those vocabularies are gone. What remains is the +enrollment lifecycle, the stable rejection reasons, and the retention vocabulary +(re-exported from the study protocol so identity and retention share one +published contract). + +Members are part of persisted contracts, so they stay additive. +""" + +from __future__ import annotations + +from enum import Enum + +from research.study.protocol.enums import RetentionAction # noqa: F401 - intentional re-export + +__all__ = [ + "EnrollmentStatus", + "IdentityReasonCode", + "RetentionAction", + "RetentionJobState", + "RetentionReasonCode", + "RetentionState", + "RetentionStepOutcome", +] + + +class EnrollmentStatus(str, Enum): + """Lifecycle of one participant's enrollment in one study revision. + + ``ACTIVE`` is the only state in which telemetry may be accepted. A study that + ends (or is terminated) marks its enrollments ``COMPLETED``, which is + terminal: the enrollment keeps its assignment but accepts nothing new. + """ + + ACTIVE = "ACTIVE" + COMPLETED = "COMPLETED" + + +class IdentityReasonCode(str, Enum): + """Stable machine-readable reason an identity operation was rejected.""" + + DUPLICATE_ENROLLMENT = "DUPLICATE_ENROLLMENT" + # One active enrollment per account across the platform: another study is + # already live for this participant. + ALREADY_ENROLLED = "ALREADY_ENROLLED" + # A completed study is terminal; the account may join a different study. + REJOIN_NOT_ALLOWED = "REJOIN_NOT_ALLOWED" + INVALID_STATE_TRANSITION = "INVALID_STATE_TRANSITION" + MAPPING_FAILURE = "MAPPING_FAILURE" + UNKNOWN_REVISION = "UNKNOWN_REVISION" + NOT_ACTIVE = "NOT_ACTIVE" + REVOKED = "REVOKED" + + # Eligibility. + ELIGIBLE = "ELIGIBLE" + INELIGIBLE = "INELIGIBLE" + UNKNOWN_ELIGIBILITY = "UNKNOWN_ELIGIBILITY" + + +class RetentionJobState(str, Enum): + """Lifecycle of one idempotent retention job. + + ``PENDING`` is enqueued when a study ends; ``RUNNING`` is written before any + data is touched; ``COMPLETED`` is terminal and never re-executed. A partial + failure is never ``COMPLETED``: it is ``RETRYABLE`` while attempts remain and + ``FAILED`` once the retry budget is exhausted. + """ + + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + RETRYABLE = "RETRYABLE" + + +class RetentionStepOutcome(str, Enum): + """What one planned retention step did (or will do) to one item.""" + + DELETED = "DELETED" + ANONYMIZED = "ANONYMIZED" + RETAINED = "RETAINED" + INVALIDATED = "INVALIDATED" + SKIPPED = "SKIPPED" + FAILED = "FAILED" + + +class RetentionReasonCode(str, Enum): + """Stable machine-readable reason for a job outcome (content-free).""" + + OK = "OK" + ALREADY_COMPLETED = "ALREADY_COMPLETED" + ACTION_MISMATCH = "ACTION_MISMATCH" + PARTIAL_FAILURE = "PARTIAL_FAILURE" + RETRY_EXHAUSTED = "RETRY_EXHAUSTED" + STORE_UNAVAILABLE = "STORE_UNAVAILABLE" + NO_ENROLLMENT = "NO_ENROLLMENT" + + +class RetentionState(str, Enum): + """Persisted per-event retention posture (``ResearchEvent.retention_state``). + + ``ANONYMIZED`` events have account-linkable payload/provenance fields + stripped. ``DELETED`` rows are content-free tombstones that read models must + exclude. + """ + + RETAINED = "RETAINED" + ANONYMIZED = "ANONYMIZED" + DELETED = "DELETED" diff --git a/src/research/participants/identity.py b/src/research/participants/identity.py new file mode 100644 index 0000000..42a15e6 --- /dev/null +++ b/src/research/participants/identity.py @@ -0,0 +1,937 @@ +"""Participant identity and retention persistence. + +Pure enrollment and retention services plus the caller-supplied-session +persistence helpers and the researcher-safe projection. The invariants enforced +here are: + +* Telemetry is accepted only for an ``ACTIVE`` enrollment; every other state + (including unknown/absent) fails closed. +* Consent is a single acceptance recorded once at join (``consent_accepted_at``): + there is no document identity, re-consent or withdrawal. +* One account has at most one ACTIVE enrollment across the platform. +* A study that ends marks its enrollments ``COMPLETED`` and bumps the revocation + epoch so previously issued capabilities stop working. +* Retention actions act on pseudonymous records and always emit a ledger entry. + +The persistence helpers take a caller-managed SQLAlchemy ``Session`` so the core +package never imports ``App`` or touches the application singleton. The router +is responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``) and for access control over the private mapping. +""" + +from __future__ import annotations + +import secrets +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError + +from database.research_schemas import ( + RECORD_KIND_RETENTION_EVIDENCE, + ResearchEnrollment, + ResearchParticipant, + ResearchRecord, + ResearchRetentionJob, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.canonical import canonical_hash +from research.runtime.sessions.enums import ( + CloseReason, + SessionReasonCode, + SessionState, +) +from research.runtime.sessions.models import SessionTransition +from research.study.protocol.enums import RetentionAction + +from .enums import ( + EnrollmentStatus, + IdentityReasonCode, +) +from .models import ( + DeletionLedgerEntry, + Enrollment, + EnrollmentResult, + IdentityIssue, + Participant, + PseudonymousRecord, + ResearchEligibility, + RetentionResult, + RevisionRef, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from research.study.protocol.publication import StudyRevision + +# Audit action used for the append-only deletion/retention ledger. +LEDGER_AUDIT_ACTION = "retention.deletion_ledger" + +# A duplicate enrollment request for the same revision returns the existing +# ACTIVE record rather than creating a second enrollment. +_REUSABLE_ENROLLMENT_STATES = frozenset({EnrollmentStatus.ACTIVE}) + +_ACTIVE_SESSION_STATES = ( + SessionState.NOT_STARTED.value, + SessionState.RUNNING.value, + SessionState.OFFLINE.value, + SessionState.SUSPENDED.value, +) + + +def _now(now: Optional[datetime] = None) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: IdentityReasonCode, message: str, field: str = "") -> IdentityIssue: + return IdentityIssue(code=code, message=message, field=field) + + +def _rejected(code: IdentityReasonCode, message: str, field: str = "") -> EnrollmentResult: + return EnrollmentResult(accepted=False, issue=_issue(code, message, field)) + + +def _isoformat(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None + + +# --------------------------------------------------------------------------- +# Enrollment state machine +# --------------------------------------------------------------------------- + + +def generate_participant_code(*, byte_length: int = 12, prefix: str = "p") -> str: + """Return a random, opaque, study-local participant code. + + The code is drawn from a CSPRNG and is not derived from, nor reversible to, + the account identity. It carries no demographic or study semantics. + """ + return f"{prefix}_{secrets.token_hex(byte_length)}" + + +def default_eligibility(now: Optional[datetime] = None) -> ResearchEligibility: + """Return the permissive default eligibility result.""" + return ResearchEligibility( + eligible=True, + reasons=[IdentityReasonCode.ELIGIBLE], + evaluated_at=_now(now), + ) + + +def revision_ref_from_mapping( + protocol_json: Mapping, + *, + revision_id: UUID, + study_id: UUID, +) -> RevisionRef: + """Build the enrollment slice from a revision protocol document.""" + privacy = protocol_json.get("privacy_policy") or {} + raw_action = privacy.get("retention_action") + try: + retention = ( + RetentionAction(raw_action) + if raw_action + else RetentionAction.RETAIN_ANONYMIZED + ) + except ValueError: + retention = RetentionAction.RETAIN_ANONYMIZED + + return RevisionRef( + revision_id=revision_id, + study_id=study_id, + retention_action=retention, + ) + + +def revision_ref_from_study_revision(revision: StudyRevision) -> RevisionRef: + """Build the enrollment slice from a published ``StudyRevision``.""" + return revision_ref_from_mapping( + revision.protocol_json, + revision_id=revision.revision_id, + study_id=revision.study_id, + ) + + +def enroll( + participant: Participant, + revision: RevisionRef, + *, + existing_enrollment: Optional[Enrollment] = None, + other_active_enrollment: Optional[Enrollment] = None, + eligibility: Optional[ResearchEligibility] = None, + now: Optional[datetime] = None, +) -> EnrollmentResult: + """Create (or reuse) a study-local enrollment for ``participant``. + + Joining records the single consent acceptance (``consent_accepted_at``) and + creates an ACTIVE enrollment directly: there is no pending-consent, + re-consent or withdrawal state. A duplicate request for the same + ``(participant, revision)`` returns the existing enrollment. A second active + enrollment is never created: ``other_active_enrollment`` is the participant's + live enrollment in another study (resolved under a participant lock by the + caller), and is rejected with a typed ``ALREADY_ENROLLED`` naming the other + study. + """ + timestamp = _now(now) + + if revision is None: + return _rejected( + IdentityReasonCode.UNKNOWN_REVISION, + "enrollment requires a published revision", + "revision_id", + ) + + if existing_enrollment is not None: + if ( + existing_enrollment.study_revision_id == revision.revision_id + and existing_enrollment.status in _REUSABLE_ENROLLMENT_STATES + ): + return EnrollmentResult( + accepted=True, + enrollment=existing_enrollment, + created=False, + reused=True, + ) + return _rejected( + IdentityReasonCode.DUPLICATE_ENROLLMENT, + ( + "an existing enrollment for this revision is not reusable " + f"({existing_enrollment.status.value})" + ), + "study_revision_id", + ) + + # One active enrollment account-wide. Completion of another study frees + # the slot, so only the live state counts. + if ( + other_active_enrollment is not None + and other_active_enrollment.status in _REUSABLE_ENROLLMENT_STATES + ): + return _rejected( + IdentityReasonCode.ALREADY_ENROLLED, + ( + "this account already has an active enrollment in study " + f"{other_active_enrollment.study_id}" + ), + "enrollment_id", + ) + + result_eligibility = eligibility or default_eligibility(timestamp) + if not result_eligibility.eligible: + return _rejected( + IdentityReasonCode.INELIGIBLE, + "this account is not eligible for the study", + "eligibility", + ) + + enrollment = Enrollment( + enrollment_id=uuid.uuid4(), + participant_id=participant.participant_id, + study_id=revision.study_id, + study_revision_id=revision.revision_id, + participant_code=generate_participant_code(), + status=EnrollmentStatus.ACTIVE, + eligibility=result_eligibility, + enrolled_at=timestamp, + consent_accepted_at=timestamp, + revocation_epoch=0, + updated_at=timestamp, + retention_action=revision.retention_action, + ) + return EnrollmentResult(accepted=True, enrollment=enrollment, created=True) + + +def apply_retention( + enrollment: Enrollment, + records: Sequence[PseudonymousRecord], + action: RetentionAction, + *, + now: Optional[datetime] = None, +) -> RetentionResult: + """Apply a retention action to pseudonymous records and emit a ledger row. + + * ``RETAIN_ANONYMIZED`` keeps every record but strips account-linkable + fields. + * ``DELETE_IDENTIFIABLE`` removes records that still carry account-linkable + fields and keeps already-pseudonymous records. + * ``DELETE_ALL`` removes every record. + """ + projected: list[PseudonymousRecord] = [] + affected = 0 + + for record in records: + linkable = ( + record.account_id is not None + or record.email is not None + or record.session_token is not None + ) + + if action == RetentionAction.RETAIN_ANONYMIZED: + if linkable: + affected += 1 + projected.append( + record.model_copy( + update={"account_id": None, "email": None, "session_token": None} + ) + ) + elif action == RetentionAction.DELETE_IDENTIFIABLE: + if linkable: + affected += 1 + continue + projected.append(record) + elif action == RetentionAction.DELETE_ALL: + affected += 1 + continue + else: # pragma: no cover - enum is closed + continue + + ledger = DeletionLedgerEntry( + ledger_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + action=action, + applied_at=_now(now), + affected_count=affected, + evidence_digest=canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment.enrollment_id), + "remaining": [record.model_dump(mode="json") for record in projected], + } + ), + ) + return RetentionResult(action=action, records=projected, ledger=ledger) + + +# --------------------------------------------------------------------------- +# Researcher-safe projection +# --------------------------------------------------------------------------- + + +def researcher_projection(enrollment: Enrollment) -> dict[str, Any]: + """Return the non-identifying, researcher-visible view of an enrollment. + + Only study-local identifiers, status and timestamps are included. The + account id, participant id, email and session tokens are deliberately + absent. + """ + return { + "enrollment_id": str(enrollment.enrollment_id), + "participant_code": enrollment.participant_code, + "study_id": str(enrollment.study_id), + "study_revision_id": str(enrollment.study_revision_id), + "status": enrollment.status.value, + "eligible": enrollment.eligibility.eligible, + "revocation_epoch": enrollment.revocation_epoch, + "retention_action": enrollment.retention_action.value, + "enrolled_at": _isoformat(enrollment.enrolled_at), + "updated_at": _isoformat(enrollment.updated_at), + } + + +# --------------------------------------------------------------------------- +# Persistence helpers (caller-supplied session) +# --------------------------------------------------------------------------- + + +def _merge_job_evidence( + session: Session, + *, + enrollment_id: uuid.UUID, + action: RetentionAction, + key: str, + payload: dict[str, Any], +) -> None: + """Merge evidence onto the retention job for ``(enrollment, action)``. + + Policy/verification evidence belongs to the job so it survives even if the + enrollment row is later removed. + """ + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + job = session.execute(statement).scalars().first() + if not isinstance(job, ResearchRetentionJob): + return + evidence = job.evidence_json if isinstance(job.evidence_json, dict) else {} + merged = dict(evidence) + merged[key] = payload + job.evidence_json = merged + session.add(job) + + +def create_participant(session: Session, participant: Participant) -> ResearchParticipant: + """Insert the private account-to-participant mapping.""" + row = ResearchParticipant( + participant_id=participant.participant_id, + account_id=participant.account_id, + created_at=participant.created_at or _now(), + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +class ActiveEnrollmentConflict(Exception): + """Raised when the account-wide active-enrollment index would be violated. + + A typed conflict rather than a raw ``IntegrityError``, so the router can + report ``ALREADY_ENROLLED`` (409) naming the other enrollment instead of a + 500. + """ + + +def lock_participant_by_account( + session: Session, account_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch the account's participant row with ``SELECT ... FOR UPDATE``. + + Taking this lock before the account-wide active-enrollment check is what + serializes two concurrent cross-study joins for the same account. + """ + statement = ( + select(ResearchParticipant) + .where(ResearchParticipant.account_id == account_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def get_or_create_participant_row( + session: Session, account_id: uuid.UUID, *, now: Optional[datetime] = None +) -> ResearchParticipant: + """Return the account's participant row, creating it race-safely and durably. + + Concurrent first-use uses ``INSERT ... ON CONFLICT DO NOTHING`` on the unique + ``account_id`` and then re-reads the winning row, so exactly one mapping + exists. The mapping is committed so it is durable and visible to the other + first-use transactions (a caller that needs the row lock re-acquires it with + :func:`lock_participant_by_account`). + """ + row = session.execute( + select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + ).scalars().first() + if row is None: + session.execute( + pg_insert(ResearchParticipant) + .values( + participant_id=uuid.uuid4(), + account_id=account_id, + created_at=_now(now), + ) + .on_conflict_do_nothing(index_elements=["account_id"]) + ) + session.commit() + row = session.execute( + select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + ).scalars().first() + if row is None: # pragma: no cover - the insert above guarantees the row + raise RuntimeError("participant mapping disappeared after an upsert") + return row + + +def get_active_enrollment_for_participant( + session: Session, + participant_id: uuid.UUID, + *, + exclude_enrollment_id: Optional[uuid.UUID] = None, +) -> Optional[ResearchEnrollment]: + """Return the account's live (active/pending/paused) enrollment, if any.""" + live_states = tuple(state.value for state in _REUSABLE_ENROLLMENT_STATES) + statement = select(ResearchEnrollment).where( + ResearchEnrollment.participant_id == participant_id, + ResearchEnrollment.status.in_(live_states), + ) + if exclude_enrollment_id is not None: + statement = statement.where( + ResearchEnrollment.enrollment_id != exclude_enrollment_id + ) + return session.execute(statement).scalars().first() + + +def create_enrollment(session: Session, enrollment: Enrollment) -> ResearchEnrollment: + """Insert an enrollment row, translating the one-active-index conflict. + + The partial unique index ``uq_research_enrollment_active_participant`` is the + database guard; a violation (e.g. a race that slipped past the participant + lock) is reported as a typed :class:`ActiveEnrollmentConflict`, never a 500. + """ + row = ResearchEnrollment( + enrollment_id=enrollment.enrollment_id, + participant_id=enrollment.participant_id, + study_id=enrollment.study_id, + study_revision_id=enrollment.study_revision_id, + participant_code=enrollment.participant_code, + status=enrollment.status.value, + revocation_epoch=enrollment.revocation_epoch, + eligibility_json=enrollment.eligibility.model_dump(mode="json"), + enrolled_at=enrollment.enrolled_at, + updated_at=enrollment.updated_at, + consent_accepted_at=enrollment.consent_accepted_at, + retention_action=enrollment.retention_action.value, + ) + try: + with session.begin_nested(): + session.add(row) + except IntegrityError as error: + raise ActiveEnrollmentConflict( + "this account already has an active enrollment" + ) from error + session.commit() + session.refresh(row) + return row + + +def get_participant( + session: Session, participant_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch a participant mapping by id, or ``None``.""" + return session.get(ResearchParticipant, participant_id) + + +def get_participant_by_account( + session: Session, account_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch the private mapping for a login account, or ``None``.""" + statement = select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + return session.execute(statement).scalars().first() + + +def get_enrollment( + session: Session, enrollment_id: uuid.UUID, *, for_update: bool = False +) -> Optional[ResearchEnrollment]: + """Fetch an enrollment by id, or ``None``. + + ``for_update`` takes a row lock (``SELECT ... FOR UPDATE``) so a caller can + serialize with a concurrent transaction that touches the same enrollment. + """ + if not for_update: + return session.get(ResearchEnrollment, enrollment_id) + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.enrollment_id == enrollment_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def get_enrollment_for_participant_revision( + session: Session, participant_id: uuid.UUID, study_revision_id: uuid.UUID +) -> Optional[ResearchEnrollment]: + """Fetch the unique enrollment for ``(participant, revision)``, or ``None``.""" + statement = select(ResearchEnrollment).where( + ResearchEnrollment.participant_id == participant_id, + ResearchEnrollment.study_revision_id == study_revision_id, + ) + return session.execute(statement).scalars().first() + + +def get_enrollment_for_participant_study( + session: Session, participant_id: uuid.UUID, study_id: uuid.UUID +) -> Optional[ResearchEnrollment]: + """Fetch the participant's newest enrollment in one study, or ``None``. + + Used to refuse a self-service rejoin to a study the account already withdrew + from or completed, including via a different revision of that study. + """ + statement = ( + select(ResearchEnrollment) + .where( + ResearchEnrollment.participant_id == participant_id, + ResearchEnrollment.study_id == study_id, + ) + .order_by(ResearchEnrollment.enrolled_at.desc()) + ) + return session.execute(statement).scalars().first() + + +#: Terminal states from which a self-service rejoin to the same study is refused. +_TERMINAL_ENROLLMENT_STATES = frozenset({EnrollmentStatus.COMPLETED}) + + +@dataclass(frozen=True) +class EnrollmentOpenResult: + """Result of the shared account-wide enrollment entry point.""" + + participant: Optional[Participant] + enrollment: Optional[Enrollment] + created: bool + reused: bool + issue: Optional[IdentityIssue] + + +def open_enrollment( + session: Session, + account_id: uuid.UUID, + revision: RevisionRef, + *, + now: Optional[datetime] = None, +) -> EnrollmentOpenResult: + """Create or reuse an enrollment under the account-wide single-active rule. + + This is the single shared entry point used by both the join-code and the + participants routers. It locks the participant row first, so concurrent + cross-study joins serialize, then rejects: + + * a second live enrollment for the account (``ALREADY_ENROLLED``); + * a rejoin to a study the account already withdrew from or completed + (``REJOIN_NOT_ALLOWED``), including via a different revision. + + Repeat enrollment in the same live revision is idempotent. + """ + timestamp = _now(now) + participant_row = get_or_create_participant_row(session, account_id, now=timestamp) + # Commit the mapping durably, then hold a row lock for the whole + # account-wide check-and-insert so concurrent cross-study joins serialize. + locked_row = lock_participant_by_account(session, account_id) + if locked_row is not None: + participant_row = locked_row + session.flush() + participant = row_to_participant(participant_row) + + prior_study_row = get_enrollment_for_participant_study( + session, participant.participant_id, revision.study_id + ) + if prior_study_row is not None: + prior_status = EnrollmentStatus(prior_study_row.status) + if prior_status in _TERMINAL_ENROLLMENT_STATES: + return EnrollmentOpenResult( + participant=participant, + enrollment=None, + created=False, + reused=False, + issue=_issue( + IdentityReasonCode.REJOIN_NOT_ALLOWED, + "this account already left or completed this study; rejoin is " + "not available", + "enrollment_id", + ), + ) + + existing_row = get_enrollment_for_participant_revision( + session, participant.participant_id, revision.revision_id + ) + existing = row_to_enrollment(existing_row) if existing_row is not None else None + + other_row = get_active_enrollment_for_participant( + session, + participant.participant_id, + exclude_enrollment_id=existing.enrollment_id if existing is not None else None, + ) + other = row_to_enrollment(other_row) if other_row is not None else None + + result = enroll( + participant, + revision, + existing_enrollment=existing, + other_active_enrollment=other, + now=timestamp, + ) + if not result.accepted or result.enrollment is None: + return EnrollmentOpenResult( + participant=participant, + enrollment=None, + created=False, + reused=False, + issue=result.issue, + ) + + if result.created: + try: + create_enrollment(session, result.enrollment) + except ActiveEnrollmentConflict: + return EnrollmentOpenResult( + participant=participant, + enrollment=None, + created=False, + reused=False, + issue=_issue( + IdentityReasonCode.ALREADY_ENROLLED, + "this account already has an active enrollment", + "enrollment_id", + ), + ) + return EnrollmentOpenResult( + participant=participant, + enrollment=result.enrollment, + created=result.created, + reused=result.reused, + issue=None, + ) + + +def list_enrollments( + session: Session, participant_id: uuid.UUID +) -> Sequence[ResearchEnrollment]: + """List a participant's enrollments, oldest-first.""" + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.participant_id == participant_id) + .order_by(ResearchEnrollment.enrolled_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def _apply_enrollment_fields( + row: ResearchEnrollment, enrollment: Enrollment +) -> None: + row.status = enrollment.status.value + row.revocation_epoch = enrollment.revocation_epoch + row.updated_at = enrollment.updated_at + row.retention_action = enrollment.retention_action.value + row.eligibility_json = enrollment.eligibility.model_dump(mode="json") + + +def update_enrollment( + session: Session, enrollment: Enrollment +) -> Optional[ResearchEnrollment]: + """Persist a status/epoch change onto an existing enrollment row.""" + row = session.get(ResearchEnrollment, enrollment.enrollment_id) + if row is None: + return None + _apply_enrollment_fields(row, enrollment) + session.commit() + session.refresh(row) + return row + + +def lock_enrollment( + session: Session, enrollment_id: uuid.UUID +) -> Optional[ResearchEnrollment]: + """Fetch an enrollment row with a ``SELECT ... FOR UPDATE`` row lock. + + The lock serializes concurrent enrollment transitions for one + enrollment; the caller owns the surrounding transaction and must commit (or + roll back) it. + """ + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.enrollment_id == enrollment_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def revoke_active_sessions( + session: Session, enrollment_id: uuid.UUID, *, now: datetime +) -> int: + """Mark every non-terminal session of an enrollment ``REVOKED``. + + Records a transition on each session and does not commit, so it can join the + caller's single transaction. + """ + statement = select(ResearchSessionRow).where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.state.in_(_ACTIVE_SESSION_STATES), + ) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + from_state = SessionState(row.state) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=row.session_id, + from_state=from_state, + to_state=SessionState.REVOKED, + occurred_at=now, + reason=SessionReasonCode.REVOKED, + close_reason=CloseReason.REVOKED, + evidence_ref="study_end", + ) + existing = row.transitions_json + serialized = list(existing) if isinstance(existing, list) else [] + serialized.append(transition.model_dump(mode="json")) + row.transitions_json = serialized + row.state = SessionState.REVOKED.value + row.last_activity_at = now + row.closed_at = now + row.close_reason = CloseReason.REVOKED.value + session.add(row) + return len(rows) + + +def complete_enrollments_for_study( + session: Session, study_id: uuid.UUID, *, now: Optional[datetime] = None +) -> int: + """Mark a study's ACTIVE enrollments COMPLETED and revoke their sessions. + + This is the terminal transition for study end/expiry: the enrollment becomes + ``COMPLETED`` (terminal), its revocation epoch is bumped so + previously issued capabilities are rejected, and every non-terminal session is + revoked. Commits once. Idempotent: already-terminal enrollments are skipped. + """ + timestamp = _now(now) + statement = select(ResearchEnrollment).where( + ResearchEnrollment.study_id == study_id, + ResearchEnrollment.status == EnrollmentStatus.ACTIVE.value, + ).with_for_update() + rows = list(session.execute(statement).scalars().all()) + for row in rows: + revoke_active_sessions(session, row.enrollment_id, now=timestamp) + row.status = EnrollmentStatus.COMPLETED.value + row.revocation_epoch = row.revocation_epoch + 1 + row.updated_at = timestamp + session.add(row) + session.commit() + return len(rows) + + +def _enqueue_retention_row( + session: Session, *, enrollment_id: uuid.UUID, action: RetentionAction, now: datetime +) -> Optional[ResearchRetentionJob]: + """Insert the PENDING retention job for ``(enrollment, action)`` if absent. + + Does not commit, so the enqueue joins the caller's transaction and the whole + workflow commits once. + """ + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + existing = session.execute(statement).scalars().first() + if existing is not None: + return existing + row = ResearchRetentionJob( + job_id=uuid.uuid4(), + enrollment_id=enrollment_id, + action=action.value, + state="PENDING", + attempts=0, + created_at=now, + affected_events=0, + evidence_json={}, + ) + session.add(row) + return row + + +def _retention_evidence_payload( + action: str, entry: DeletionLedgerEntry +) -> dict[str, Any]: + return { + "action": action, + "enrollment_id": str(entry.enrollment_id), + "record": entry.model_dump(mode="json"), + } + + +def insert_deletion_ledger( + session: Session, entry: DeletionLedgerEntry +) -> DeletionLedgerEntry: + """Append a deletion/retention ledger entry. + + The evidence is written to the generic ``research_record`` table + (``kind = RETENTION_EVIDENCE``) and, when a retention job exists for + ``(enrollment, action)``, merged into its ``evidence_json``. + """ + row = ResearchRecord( + record_id=entry.ledger_id, + kind=RECORD_KIND_RETENTION_EVIDENCE, + scope_type="enrollment", + scope_id=entry.enrollment_id, + study_id=None, + actor=None, + occurred_at=entry.applied_at, + payload_json=_retention_evidence_payload(LEDGER_AUDIT_ACTION, entry), + ) + session.add(row) + _merge_job_evidence( + session, + enrollment_id=entry.enrollment_id, + action=entry.action, + key="ledger", + payload=entry.model_dump(mode="json"), + ) + session.commit() + return entry + + +def list_deletion_ledger( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[DeletionLedgerEntry]: + """List deletion ledger entries (optionally for one enrollment).""" + statement = select(ResearchRecord).where( + ResearchRecord.kind == RECORD_KIND_RETENTION_EVIDENCE, + ResearchRecord.payload_json["action"].astext == LEDGER_AUDIT_ACTION, + ) + if enrollment_id is not None: + statement = statement.where( + ResearchRecord.payload_json["enrollment_id"].astext == str(enrollment_id) + ) + statement = statement.order_by(ResearchRecord.occurred_at.asc()) + return [ + row_to_deletion_ledger(row) + for row in session.execute(statement).scalars().all() + ] + + +def row_to_participant(row: ResearchParticipant) -> Participant: + """Rehydrate the private participant mapping.""" + from .models import Participant as ParticipantModel + + return ParticipantModel( + participant_id=row.participant_id, + account_id=row.account_id, + created_at=row.created_at, + ) + + +def row_to_enrollment(row: ResearchEnrollment) -> Enrollment: + """Rehydrate an enrollment row into the domain model.""" + from .models import Enrollment as EnrollmentModel + + status = row.status + eligibility = ResearchEligibility.model_validate(row.eligibility_json or {}) + return EnrollmentModel( + enrollment_id=row.enrollment_id, + participant_id=row.participant_id, + study_id=row.study_id, + study_revision_id=row.study_revision_id, + participant_code=row.participant_code, + status=status + if isinstance(status, EnrollmentStatus) + else EnrollmentStatus(status), + eligibility=eligibility, + enrolled_at=row.enrolled_at, + consent_accepted_at=row.consent_accepted_at, + revocation_epoch=row.revocation_epoch, + updated_at=row.updated_at, + retention_action=RetentionAction(row.retention_action), + ) + + +def row_to_deletion_ledger(row: ResearchRecord) -> DeletionLedgerEntry: + """Rehydrate a deletion ledger record row into the domain model.""" + from .models import DeletionLedgerEntry as DeletionLedgerEntryModel + + return DeletionLedgerEntryModel.model_validate(row.payload_json["record"]) + + +def deletion_ledger_summary(row: DeletionLedgerEntry) -> dict[str, Any]: + """Return a compact, non-secret deletion ledger summary.""" + applied_at = row.applied_at + action = row.action + return { + "ledger_id": str(row.ledger_id), + "enrollment_id": str(row.enrollment_id), + "action": action.value if isinstance(action, RetentionAction) else action, + "applied_at": ( + applied_at.isoformat() if isinstance(applied_at, datetime) else None + ), + "affected_count": row.affected_count, + "evidence_digest": row.evidence_digest, + } diff --git a/src/research/participants/models.py b/src/research/participants/models.py new file mode 100644 index 0000000..6e2d0d2 --- /dev/null +++ b/src/research/participants/models.py @@ -0,0 +1,284 @@ +"""Pydantic v2 contracts for participant identity and retention. + +Consent is a single acceptance recorded once at join: there is no document +identity, no re-consent and no withdrawal. Privacy invariant: only +:class:`Participant` carries the private account-to-participant mapping. +Enrollment and every researcher projection are keyed by opaque UUIDs and a +random study-local ``participant_code`` that is never derived from, or parsed +back to, login identity. + +No retention model carries raw event content, an account id, an email or a +session token. A plan references events/exports by opaque id and a step +outcome; the evidence digest is computed over that plan plus counts and is +reproducible without reading content. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + EnrollmentStatus, + IdentityReasonCode, + RetentionAction, + RetentionJobState, + RetentionReasonCode, + RetentionStepOutcome, +) + +_BASE_CONFIG = ConfigDict(extra="forbid") +_FROZEN_CONFIG = ConfigDict(extra="forbid", frozen=True) + +_BASE = ConfigDict(extra="forbid") +_FROZEN = ConfigDict(extra="forbid", frozen=True) + +__all__ = [ + "DeletionLedgerEntry", + "Enrollment", + "EnrollmentResult", + "IdentityIssue", + "Participant", + "PseudonymousRecord", + "ResearchEligibility", + "RetentionEventOutcome", + "RetentionEvidence", + "RetentionExecution", + "RetentionJob", + "RetentionPlan", + "RetentionResult", + "RevisionRef", +] + + +# --------------------------------------------------------------------------- +# Identity +# --------------------------------------------------------------------------- + + +class Participant(BaseModel): + """Private account-to-participant mapping. + + This is the only object that links a login account to a study participant. + It is privileged: it is never returned to researchers and is never embedded + in a researcher projection or export. + """ + + model_config = _BASE_CONFIG + + participant_id: UUID + account_id: UUID + created_at: datetime + + +class ResearchEligibility(BaseModel): + """Typed eligibility result, evaluated before enrollment.""" + + model_config = _BASE_CONFIG + + eligible: bool + reasons: list[IdentityReasonCode] = Field(default_factory=list) + evaluated_at: Optional[datetime] = None + + +class RevisionRef(BaseModel): + """The slice of a published revision that enrollment needs. + + Only the identity (revision/study) and the study's retention choice are + carried: consent has no per-study document identity. + """ + + model_config = _BASE_CONFIG + + revision_id: UUID + study_id: UUID + retention_action: RetentionAction = RetentionAction.RETAIN_ANONYMIZED + + +class Enrollment(BaseModel): + """One participant's revision-bound enrollment (the research identity).""" + + model_config = _BASE_CONFIG + + enrollment_id: UUID + participant_id: UUID + study_id: UUID + study_revision_id: UUID + # Random, opaque, study-local. Never derived from the account id and never + # parsed for meaning; it is the only participant handle researchers see. + participant_code: str + status: EnrollmentStatus = EnrollmentStatus.ACTIVE + eligibility: ResearchEligibility + enrolled_at: datetime + # Set at join: the participant accepted the global consent text once. + consent_accepted_at: Optional[datetime] = None + revocation_epoch: int = 0 + updated_at: datetime + retention_action: RetentionAction = RetentionAction.RETAIN_ANONYMIZED + + +class DeletionLedgerEntry(BaseModel): + """Append-only evidence that a retention action was applied.""" + + model_config = _BASE_CONFIG + + ledger_id: UUID + enrollment_id: UUID + action: RetentionAction + applied_at: datetime + affected_count: int + evidence_digest: str + + +class PseudonymousRecord(BaseModel): + """A research record that may carry account-linkable fields. + + ``account_id``, ``email`` and ``session_token`` are the linkable fields a + retention action strips or removes. ``participant_code`` and + ``pseudonymous_payload`` are the safe research fields. + """ + + model_config = _BASE_CONFIG + + record_id: str + enrollment_id: Optional[UUID] = None + participant_code: Optional[str] = None + account_id: Optional[UUID] = None + email: Optional[str] = None + session_token: Optional[str] = None + pseudonymous_payload: dict[str, Any] = Field(default_factory=dict) + + +class IdentityIssue(BaseModel): + """One typed identity rejection reason.""" + + model_config = _BASE_CONFIG + + code: IdentityReasonCode + message: str + field: str = "" + + +class EnrollmentResult(BaseModel): + """Typed outcome of an enrollment operation.""" + + model_config = _BASE_CONFIG + + accepted: bool + enrollment: Optional[Enrollment] = None + created: bool = False + reused: bool = False + issue: Optional[IdentityIssue] = None + + +class RetentionResult(BaseModel): + """Typed outcome of applying a retention action to pseudonymous records.""" + + model_config = _BASE_CONFIG + + action: RetentionAction + records: list[PseudonymousRecord] = Field(default_factory=list) + ledger: DeletionLedgerEntry + + +# --------------------------------------------------------------------------- +# Retention execution +# --------------------------------------------------------------------------- + + +class RetentionEventOutcome(BaseModel): + """The planned outcome for one stored event (id + step only).""" + + model_config = _FROZEN + + event_id: UUID + outcome: RetentionStepOutcome + + +class RetentionPlan(BaseModel): + """A deterministic plan for one retention action over one enrollment.""" + + model_config = _FROZEN + + enrollment_id: UUID + action: RetentionAction + event_outcomes: list[RetentionEventOutcome] = Field(default_factory=list) + plan_digest: str + + def _event_ids(self, outcome: RetentionStepOutcome) -> list[UUID]: + return [ + item.event_id + for item in self.event_outcomes + if item.outcome == outcome + ] + + def deleted_event_ids(self) -> list[UUID]: + """Event ids that will be deleted/tombstoned.""" + return self._event_ids(RetentionStepOutcome.DELETED) + + def anonymized_event_ids(self) -> list[UUID]: + """Event ids that will be kept but anonymized.""" + return self._event_ids(RetentionStepOutcome.ANONYMIZED) + + def affected_event_ids(self) -> list[UUID]: + """Event ids whose stored state changes (deleted or anonymized).""" + return self.deleted_event_ids() + self.anonymized_event_ids() + + @property + def affected_events(self) -> int: + """Number of events whose stored state changes.""" + return len(self.affected_event_ids()) + + +class RetentionJob(BaseModel): + """Durable, idempotent state machine for one enrollment/action.""" + + model_config = _BASE + + job_id: UUID + enrollment_id: UUID + action: RetentionAction + state: RetentionJobState = RetentionJobState.PENDING + attempts: int = 0 + created_at: datetime + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + last_error: Optional[str] = None + evidence_digest: Optional[str] = None + affected_events: int = 0 + reason: RetentionReasonCode = RetentionReasonCode.OK + + +class RetentionEvidence(BaseModel): + """Immutable, content-free evidence that a retention action ran. + + ``ledger_id`` is persisted as a ``research_record`` row + (``kind = RETENTION_EVIDENCE``) and merged into the matching + ``research_retention_job.evidence_json`` so an operator can verify execution + without reading any event content. + """ + + model_config = _FROZEN + + ledger_id: UUID + enrollment_id: UUID + action: RetentionAction + applied_at: datetime + affected_events: int + evidence_digest: str + + +class RetentionExecution(BaseModel): + """Result of executing (or re-executing) one retention job.""" + + model_config = _BASE + + job: RetentionJob + plan: Optional[RetentionPlan] = None + evidence: Optional[RetentionEvidence] = None + deleted_event_ids: list[UUID] = Field(default_factory=list) + anonymized_event_ids: list[UUID] = Field(default_factory=list) + reused: bool = False diff --git a/src/research/participants/retention.py b/src/research/participants/retention.py new file mode 100644 index 0000000..8ca00a9 --- /dev/null +++ b/src/research/participants/retention.py @@ -0,0 +1,923 @@ +"""Pure retention planning/execution, persistence helpers and worker entry points. + +This module is deterministic and framework-free: it never imports ``App``, +FastAPI or the ORM eagerly. Given a job and the enrollment's stored events it +produces a plan, applies the published retention action (via +:func:`apply_retention` semantics) through an +injected applier, and transitions the job state. + +The SQLAlchemy helpers take a caller-supplied ``Session`` (like every other +research store) and return ORM rows; :class:`SqlAlchemyRetentionStore` is a +thin, injectable wrapper that returns domain objects to the worker. + +The worker processes a bounded batch with retries: a job becomes ``RUNNING``, +executes, and finishes ``COMPLETED`` or ``RETRYABLE``/``FAILED``. Exceptions +never crash the process; they are recorded on the job with a content-free +evidence digest. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Iterable, Optional, Protocol, Sequence +from uuid import UUID + +from sqlalchemy import select + +from database.research_schemas import ( + ResearchAgentRun, + ResearchEnrollment, + ResearchEvent, + ResearchRetentionJob, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.canonical import canonical_hash +from research.runtime.sessions.enums import ( + AgentRunOutcome, + CloseReason, + SessionState, +) +from research.study.protocol.enums import RetentionAction +from research.telemetry.ingestion.store import row_to_record + +from .enums import ( + RetentionJobState, + RetentionReasonCode, + RetentionState, + RetentionStepOutcome, +) +from .identity import PseudonymousRecord, apply_retention, insert_deletion_ledger +from .models import ( + DeletionLedgerEntry, + RetentionEventOutcome, + RetentionEvidence, + RetentionExecution, + RetentionJob, + RetentionPlan, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from research.telemetry.ingestion.models import ResearchEventRecord + + from .models import Enrollment + +__all__ = [ + "RetentionApplier", + "SqlAlchemyRetentionStore", + "apply_event_anonymization", + "apply_event_deletion", + "enqueue_for_enrollment", + "event_to_pseudonymous_record", + "execute_retention", + "get_enrollment_study_id", + "get_retention_job", + "get_retention_job_for_enrollment_action", + "is_event_visible", + "job_summary", + "list_candidate_events", + "list_retention_jobs_for_enrollment", + "plan_retention", + "revoke_sessions", + "row_to_retention_job", + "run_retention_for_enrollment", + "run_retention_job", + "start_retention_job", + "upsert_retention_job", +] + +_LINKABLE_KEYS = ("account_id", "email", "session_token") + + +class RetentionApplier(Protocol): + """Side-effecting steps the pure executor calls (implemented by the store).""" + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: # pragma: no cover - structural protocol + ... + + def apply_session_retention( + self, + enrollment_id: UUID, + *, + now: datetime, + ) -> None: # pragma: no cover - structural protocol + ... + + +class _NullApplier: + """Default in-memory applier: plans but persists nothing.""" + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: + return None + + def apply_session_retention( + self, + enrollment_id: UUID, + *, + now: datetime, + ) -> None: + return None + + +class _EnrollmentRef: + """Minimal object carrying the one attribute ``apply_retention`` reads.""" + + __slots__ = ("enrollment_id",) + + def __init__(self, enrollment_id: UUID) -> None: + self.enrollment_id = enrollment_id + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _coerce_uuid(value: Any) -> Optional[UUID]: + if value is None: + return None + if isinstance(value, UUID): + return value + try: + return UUID(str(value)) + except (TypeError, ValueError, AttributeError): + return None + + +def _envelope(event: Any) -> dict[str, Any]: + envelope = getattr(event, "envelope", None) + return dict(envelope) if isinstance(envelope, dict) else {} + + +def _payload(event: Any) -> dict[str, Any]: + # Stored records carry content only inside their canonical envelope; a + # canonical event exposes ``payload`` directly. + payload = _envelope(event).get("payload") + if not isinstance(payload, dict): + payload = getattr(event, "payload", None) + return dict(payload) if isinstance(payload, dict) else {} + + +def _provenance(event: Any) -> dict[str, Any]: + provenance = _envelope(event).get("provenance") + if not isinstance(provenance, dict): + provenance = getattr(event, "provenance", None) + return dict(provenance) if isinstance(provenance, dict) else {} + + +def _extract_linkable(event: Any) -> dict[str, Any]: + """Return the present account-linkable fields from payload/provenance. + + Only the presence of a value matters for linkability; the values are never + logged or persisted by this module. + """ + payload = _payload(event) + provenance = _provenance(event) + found: dict[str, Any] = {} + for key in _LINKABLE_KEYS: + value = payload.get(key) + if value is None: + value = provenance.get(key) + if value is not None: + found[key] = value + return found + + +def is_event_visible(event: Any) -> bool: + """Whether a stored event is still part of read models and exports. + + A ``DELETED`` tombstone is never visible; ``ANONYMIZED`` and ``RETAINED`` + events remain exportable. + """ + state = getattr(event, "retention_state", RetentionState.RETAINED.value) + return state != RetentionState.DELETED.value + + +def event_to_pseudonymous_record(event: Any) -> PseudonymousRecord: + """Project one stored event into the identity retention vocabulary. + + Only ``account_id``, ``email`` and ``session_token`` count as linkable, so + a normal pseudonymous event is a no-op for ``DELETE_IDENTIFIABLE`` while a + planted linkable payload is removed or stripped exactly as + :func:`apply_retention` specifies. + """ + payload = _payload(event) + provenance = _provenance(event) + account_raw = payload.get("account_id") + if account_raw is None: + account_raw = provenance.get("account_id") + email_raw = payload.get("email") + if email_raw is None: + email_raw = provenance.get("email") + token_raw = payload.get("session_token") + if token_raw is None: + token_raw = provenance.get("session_token") + + pseudonymous_payload = { + key: value + for key, value in payload.items() + if key not in _LINKABLE_KEYS + } + return PseudonymousRecord( + record_id=str(event.event_id), + enrollment_id=getattr(event, "enrollment_id", None), + participant_code=None, + account_id=_coerce_uuid(account_raw), + email=str(email_raw) if email_raw is not None else None, + session_token=str(token_raw) if token_raw is not None else None, + pseudonymous_payload=pseudonymous_payload, + ) + + +def _plan_digest( + enrollment_id: UUID, + action: RetentionAction, + event_outcomes: Sequence[RetentionEventOutcome], +) -> str: + return canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment_id), + "events": sorted( + (str(item.event_id), item.outcome.value) for item in event_outcomes + ), + } + ) + + +def _evidence_digest(plan: RetentionPlan) -> str: + return canonical_hash( + { + "plan_digest": plan.plan_digest, + "action": plan.action.value, + "enrollment_id": str(plan.enrollment_id), + "affected_events": plan.affected_events, + } + ) + + +def plan_retention( + enrollment: Enrollment, + action: RetentionAction, + records: Iterable[ResearchEventRecord], + *, + now: Optional[datetime] = None, +) -> RetentionPlan: + """Return a deterministic plan for one enrollment and retention action. + + The event outcomes reuse :func:`apply_retention` + so ``DELETE_ALL``, ``DELETE_IDENTIFIABLE`` and ``RETAIN_ANONYMIZED`` mean + exactly what they mean for pseudonymous records. + """ + timestamp = _now(now) + events = list(records) + pseudonyms = [event_to_pseudonymous_record(event) for event in events] + result = apply_retention(enrollment, pseudonyms, action, now=timestamp) + retained_ids = {record.record_id for record in result.records} + + event_outcomes: list[RetentionEventOutcome] = [] + for event in events: + record_id = str(event.event_id) + if record_id not in retained_ids: + outcome = RetentionStepOutcome.DELETED + elif action == RetentionAction.RETAIN_ANONYMIZED and _extract_linkable(event): + outcome = RetentionStepOutcome.ANONYMIZED + else: + outcome = RetentionStepOutcome.RETAINED + event_outcomes.append( + RetentionEventOutcome(event_id=event.event_id, outcome=outcome) + ) + + return RetentionPlan( + enrollment_id=enrollment.enrollment_id, + action=action, + event_outcomes=event_outcomes, + plan_digest=_plan_digest(enrollment.enrollment_id, action, event_outcomes), + ) + + +def start_retention_job( + job: RetentionJob, *, now: Optional[datetime] = None +) -> RetentionJob: + """Transition a job to ``RUNNING`` and increment its attempt counter.""" + timestamp = _now(now) + return job.model_copy( + update={ + "state": RetentionJobState.RUNNING, + "attempts": job.attempts + 1, + "started_at": job.started_at or timestamp, + "completed_at": None, + } + ) + + +def _build_evidence( + job: RetentionJob, plan: RetentionPlan, timestamp: datetime +) -> RetentionEvidence: + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=timestamp, + affected_events=plan.affected_events, + evidence_digest=_evidence_digest(plan), + ) + + +def _evidence_from_job(job: RetentionJob, timestamp: datetime) -> RetentionEvidence: + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=job.completed_at or timestamp, + affected_events=job.affected_events, + evidence_digest=job.evidence_digest or "", + ) + + +def execute_retention( + job: RetentionJob, + events: Iterable[ResearchEventRecord], + action: RetentionAction, + now: Optional[datetime] = None, + *, + applier: Optional[RetentionApplier] = None, + max_attempts: int = 3, +) -> RetentionExecution: + """Execute (or re-execute) one retention job. + + * A ``COMPLETED`` job is a no-op returning the same job and evidence digest. + * A ``FAILED``/``RETRYABLE``/``PENDING`` job is planned and executed; the + attempt counter starts at the current value. + * On an applier failure the job becomes ``RETRYABLE`` while attempts remain + and ``FAILED`` afterwards; it is never ``COMPLETED`` and the plan/evidence + is returned so the caller can retain the ledger. + """ + timestamp = _now(now) + + if job.state == RetentionJobState.COMPLETED: + return RetentionExecution( + job=job, reused=True, evidence=_evidence_from_job(job, timestamp) + ) + + if action != job.action: + failed = job.model_copy( + update={ + "state": RetentionJobState.FAILED, + "started_at": job.started_at or timestamp, + "completed_at": timestamp, + "last_error": RetentionReasonCode.ACTION_MISMATCH.value, + "reason": RetentionReasonCode.ACTION_MISMATCH, + } + ) + return RetentionExecution(job=failed, reused=False) + + running = ( + job + if job.state == RetentionJobState.RUNNING + else start_retention_job(job, now=timestamp) + ) + plan = plan_retention( + _EnrollmentRef(running.enrollment_id), + action, + list(events), + now=timestamp, + ) + evidence = _build_evidence(running, plan, timestamp) + deleted = plan.deleted_event_ids() + anonymized = plan.anonymized_event_ids() + active_applier: RetentionApplier = applier or _NullApplier() + + try: + # DELETE_ALL also revokes the enrollment's sessions/runs, so a later + # re-export that reads sessions independently of events cannot resurrect + # the deleted participant's activity. + if action == RetentionAction.DELETE_ALL: + active_applier.apply_session_retention( + running.enrollment_id, now=timestamp + ) + if deleted or anonymized: + active_applier.apply_event_retention(deleted, anonymized, now=timestamp) + except Exception as error: # noqa: BLE001 - any step failure is a job failure + retryable = running.attempts < max_attempts + state = ( + RetentionJobState.RETRYABLE if retryable else RetentionJobState.FAILED + ) + reason = ( + RetentionReasonCode.PARTIAL_FAILURE + if retryable + else RetentionReasonCode.RETRY_EXHAUSTED + ) + failed_job = running.model_copy( + update={ + "state": state, + "completed_at": None, + "last_error": type(error).__name__, + "evidence_digest": evidence.evidence_digest, + "affected_events": plan.affected_events, + "reason": reason, + } + ) + return RetentionExecution( + job=failed_job, + plan=plan, + evidence=evidence, + deleted_event_ids=deleted, + anonymized_event_ids=anonymized, + reused=False, + ) + + completed = running.model_copy( + update={ + "state": RetentionJobState.COMPLETED, + "completed_at": timestamp, + "last_error": None, + "evidence_digest": evidence.evidence_digest, + "affected_events": plan.affected_events, + "reason": RetentionReasonCode.OK, + } + ) + return RetentionExecution( + job=completed, + plan=plan, + evidence=evidence, + deleted_event_ids=deleted, + anonymized_event_ids=anonymized, + reused=False, + ) + + +# --------------------------------------------------------------------------- +# SQLAlchemy helpers (session supplied by the caller) +# --------------------------------------------------------------------------- + + +def get_enrollment_study_id( + session: Session, enrollment_id: UUID +) -> Optional[UUID]: + """Return the study a stored enrollment belongs to, or ``None``.""" + row = session.get(ResearchEnrollment, enrollment_id) + return row.study_id if row is not None else None + + +def list_candidate_events( + session: Session, enrollment_id: UUID +) -> list[ResearchEventRecord]: + """Load an enrollment's non-tombstoned events as domain records.""" + statement = ( + select(ResearchEvent) + .where( + ResearchEvent.enrollment_id == enrollment_id, + ResearchEvent.retention_state != RetentionState.DELETED.value, + ) + .order_by(ResearchEvent.occurred_at.asc(), ResearchEvent.event_id.asc()) + ) + return [ + row_to_record(row) + for row in session.execute(statement).scalars().all() + ] + + +def _clear_event_content(row: ResearchEvent, now: datetime) -> None: + row.retention_state = RetentionState.DELETED.value + row.anonymized_at = now + row.enrollment_id = None + row.research_session_id = None + row.agent_run_id = None + # The complete envelope is content-free too: no copy of the deleted content + # may survive in any column or in the envelope. + row.envelope_json = {} + + +def apply_event_deletion( + session: Session, event_ids: Sequence[UUID], now: datetime +) -> int: + """Tombstone events: clear all content and mark them deleted.""" + ids = list(event_ids) + if not ids: + return 0 + statement = select(ResearchEvent).where(ResearchEvent.event_id.in_(ids)) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + _clear_event_content(row, now) + session.commit() + return len(rows) + + +def _strip_linkable(mapping: Any) -> dict[str, Any]: + if not isinstance(mapping, dict): + return {} + return {key: value for key, value in mapping.items() if key not in _LINKABLE_KEYS} + + +def apply_event_anonymization( + session: Session, event_ids: Sequence[UUID], now: datetime +) -> int: + """Strip account-linkable fields and mark events anonymized.""" + ids = list(event_ids) + if not ids: + return 0 + statement = select(ResearchEvent).where(ResearchEvent.event_id.in_(ids)) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + row.retention_state = RetentionState.ANONYMIZED.value + row.anonymized_at = now + # The envelope is the only content authority, so it is scrubbed in place. + envelope = dict(getattr(row, "envelope_json", None) or {}) + for key in ("payload", "provenance", "correlations", "unknown_payload"): + if key in envelope: + if isinstance(envelope[key], dict): + envelope[key] = _strip_linkable(envelope[key]) + else: + envelope.pop(key, None) + row.envelope_json = envelope + session.commit() + return len(rows) + + +def revoke_sessions(session: Session, enrollment_id: UUID, now: datetime) -> int: + """Mark an enrollment's sessions (and their runs) revoked for DELETE_ALL. + + A tombstoned enrollment must not leave live session/run rows behind: a + later re-export reads sessions independently of events, so without this a + deleted participant's activity could reappear in a fresh bundle. + """ + statement = select(ResearchSessionRow).where( + ResearchSessionRow.enrollment_id == enrollment_id + ) + sessions = list(session.execute(statement).scalars().all()) + if not sessions: + return 0 + session_ids = [row.session_id for row in sessions] + for row in sessions: + row.state = SessionState.REVOKED.value + row.closed_at = now + row.close_reason = CloseReason.REVOKED.value + run_statement = select(ResearchAgentRun).where( + ResearchAgentRun.research_session_id.in_(session_ids) + ) + for run in session.execute(run_statement).scalars().all(): + if run.ended_at is None: + run.ended_at = now + run.outcome = AgentRunOutcome.REVOKED.value + session.commit() + return len(sessions) + + +def _apply_job_fields(row: ResearchRetentionJob, job: RetentionJob) -> None: + row.enrollment_id = job.enrollment_id + row.action = job.action.value + row.state = job.state.value + row.attempts = job.attempts + row.created_at = job.created_at + row.started_at = job.started_at + row.completed_at = job.completed_at + row.last_error = job.last_error + row.evidence_digest = job.evidence_digest + row.affected_events = job.affected_events + + +def upsert_retention_job( + session: Session, job: RetentionJob +) -> ResearchRetentionJob: + """Insert or update one retention job (idempotent per enrollment/action).""" + row = session.get(ResearchRetentionJob, job.job_id) + if row is None: + existing = get_retention_job_for_enrollment_action( + session, job.enrollment_id, job.action + ) + if existing is not None: + return existing + row = ResearchRetentionJob(job_id=job.job_id) + _apply_job_fields(row, job) + session.add(row) + else: + _apply_job_fields(row, job) + session.commit() + session.refresh(row) + return row + + +def get_retention_job( + session: Session, job_id: UUID +) -> Optional[ResearchRetentionJob]: + """Fetch a retention job row by id, or ``None``.""" + return session.get(ResearchRetentionJob, job_id) + + +def get_retention_job_for_enrollment_action( + session: Session, enrollment_id: UUID, action: RetentionAction +) -> Optional[ResearchRetentionJob]: + """Fetch the unique job for ``(enrollment, action)``, or ``None``.""" + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + return session.execute(statement).scalars().first() + + +def list_retention_jobs_for_enrollment( + session: Session, enrollment_id: UUID +) -> list[ResearchRetentionJob]: + """List an enrollment's retention jobs, oldest-first.""" + statement = ( + select(ResearchRetentionJob) + .where(ResearchRetentionJob.enrollment_id == enrollment_id) + .order_by(ResearchRetentionJob.created_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_retention_job(row: ResearchRetentionJob) -> RetentionJob: + """Rehydrate a retention job row into the domain model.""" + state = RetentionJobState(row.state) + if state == RetentionJobState.RETRYABLE: + reason = RetentionReasonCode.PARTIAL_FAILURE + elif state == RetentionJobState.FAILED: + reason = RetentionReasonCode.RETRY_EXHAUSTED + else: + reason = RetentionReasonCode.OK + return RetentionJob( + job_id=row.job_id, + enrollment_id=row.enrollment_id, + action=RetentionAction(row.action), + state=state, + attempts=row.attempts or 0, + created_at=row.created_at, + started_at=row.started_at, + completed_at=row.completed_at, + last_error=row.last_error, + evidence_digest=row.evidence_digest, + affected_events=row.affected_events or 0, + reason=reason, + ) + + +def job_summary(job: RetentionJob) -> dict[str, Any]: + """Compact, non-secret job summary safe for operator responses.""" + return { + "job_id": str(job.job_id), + "enrollment_id": str(job.enrollment_id), + "action": job.action.value, + "state": job.state.value, + "attempts": job.attempts, + "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), + "completed_at": _iso(job.completed_at), + "last_error": job.last_error, + "evidence_digest": job.evidence_digest, + "affected_events": job.affected_events, + "reason": job.reason.value, + } + + +def _iso(value: Any) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None + + +def insert_retention_evidence( + session: Session, evidence: RetentionEvidence +) -> Any: + """Append a content-free deletion-ledger row for an executed job.""" + entry = DeletionLedgerEntry( + ledger_id=evidence.ledger_id, + enrollment_id=evidence.enrollment_id, + action=evidence.action, + applied_at=evidence.applied_at, + affected_count=evidence.affected_events, + evidence_digest=evidence.evidence_digest, + ) + return insert_deletion_ledger(session, entry) + + +# --------------------------------------------------------------------------- +# Injectable SQLAlchemy wrapper +# --------------------------------------------------------------------------- + + +class SqlAlchemyRetentionStore: + """Adapter exposing the retention persistence surface over one session.""" + + def __init__(self, session: Session) -> None: + self.session = session + + def get_enrollment_study_id(self, enrollment_id: UUID) -> Optional[UUID]: + return get_enrollment_study_id(self.session, enrollment_id) + + def load_events(self, enrollment_id: UUID) -> list[ResearchEventRecord]: + return list_candidate_events(self.session, enrollment_id) + + def mark_events_deleted(self, event_ids: Sequence[UUID], now: datetime) -> int: + return apply_event_deletion(self.session, event_ids, now) + + def anonymize_events(self, event_ids: Sequence[UUID], now: datetime) -> int: + return apply_event_anonymization(self.session, event_ids, now) + + def revoke_sessions(self, enrollment_id: UUID, now: datetime) -> int: + return revoke_sessions(self.session, enrollment_id, now) + + def get_job(self, job_id: UUID) -> Optional[RetentionJob]: + row = get_retention_job(self.session, job_id) + return row_to_retention_job(row) if row is not None else None + + def get_job_for_enrollment_action( + self, enrollment_id: UUID, action: RetentionAction + ) -> Optional[RetentionJob]: + row = get_retention_job_for_enrollment_action( + self.session, enrollment_id, action + ) + return row_to_retention_job(row) if row is not None else None + + def upsert_job(self, job: RetentionJob) -> RetentionJob: + return row_to_retention_job(upsert_retention_job(self.session, job)) + + def insert_evidence(self, evidence: RetentionEvidence) -> None: + insert_retention_evidence(self.session, evidence) + + +# --------------------------------------------------------------------------- +# Worker entry points +# --------------------------------------------------------------------------- + + +class _StoreApplier: + """Applies a plan through the retention persistence surface.""" + + def __init__(self, api: Any) -> None: + self._api = api + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: + if deleted_event_ids: + self._api.mark_events_deleted(deleted_event_ids, now) + if anonymized_event_ids: + self._api.anonymize_events(anonymized_event_ids, now) + + def apply_session_retention(self, enrollment_id: UUID, *, now: datetime) -> None: + self._api.revoke_sessions(enrollment_id, now) + + +def _resolve_store(session: Any, store_api: Optional[Any]) -> Any: + if store_api is not None: + return store_api + if session is None: + raise ValueError("a session or an explicit retention store is required") + return SqlAlchemyRetentionStore(session) + + +def _failure_evidence( + job: RetentionJob, error: Exception, timestamp: datetime +) -> RetentionEvidence: + digest = canonical_hash( + { + "action": job.action.value, + "enrollment_id": str(job.enrollment_id), + "error": type(error).__name__, + } + ) + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=timestamp, + affected_events=job.affected_events, + evidence_digest=digest, + ) + + +def _failure_execution( + job: RetentionJob, error: Exception, *, now: datetime, max_attempts: int +) -> RetentionExecution: + retryable = job.attempts < max_attempts + state = RetentionJobState.RETRYABLE if retryable else RetentionJobState.FAILED + reason = ( + RetentionReasonCode.PARTIAL_FAILURE + if retryable + else RetentionReasonCode.RETRY_EXHAUSTED + ) + evidence = _failure_evidence(job, error, now) + failed = job.model_copy( + update={ + "state": state, + "completed_at": None, + "last_error": type(error).__name__, + "evidence_digest": evidence.evidence_digest, + "reason": reason, + } + ) + return RetentionExecution(job=failed, evidence=evidence, reused=False) + + +def enqueue_for_enrollment( + session: Any, + enrollment_id: UUID, + action: RetentionAction, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, +) -> RetentionJob: + """Get or create the single retention job for ``(enrollment, action)``.""" + api = _resolve_store(session, store_api) + existing = api.get_job_for_enrollment_action(enrollment_id, action) + if existing is not None: + return existing + job = RetentionJob( + job_id=uuid.uuid4(), + enrollment_id=enrollment_id, + action=action, + state=RetentionJobState.PENDING, + attempts=0, + created_at=now or datetime.now(timezone.utc), + ) + return api.upsert_job(job) + + +def run_retention_job( + session: Any, + job_id: UUID, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, + max_attempts: int = 3, +) -> RetentionJob: + """Execute one retention job; never raises for a step failure.""" + api = _resolve_store(session, store_api) + timestamp = now or datetime.now(timezone.utc) + job = api.get_job(job_id) + if job is None: + raise ValueError(f"retention job {job_id} not found") + if job.state == RetentionJobState.COMPLETED: + return job + + running = start_retention_job(job, now=timestamp) + api.upsert_job(running) + try: + events = api.load_events(job.enrollment_id) + execution = execute_retention( + running, + events, + job.action, + now=timestamp, + applier=_StoreApplier(api), + max_attempts=max_attempts, + ) + persisted = api.upsert_job(execution.job) + if execution.evidence is not None and not execution.reused: + api.insert_evidence(execution.evidence) + return persisted + except Exception as error: # noqa: BLE001 - any failure is recorded, not raised + execution = _failure_execution( + running, error, now=timestamp, max_attempts=max_attempts + ) + try: + api.upsert_job(execution.job) + if execution.evidence is not None: + api.insert_evidence(execution.evidence) + except Exception: # noqa: BLE001 - never crash the worker process + pass + return execution.job + + +def run_retention_for_enrollment( + session: Any, + enrollment_id: UUID, + action: RetentionAction, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, + max_attempts: int = 3, +) -> RetentionJob: + """Idempotently enqueue and execute the job for one enrollment.""" + api = _resolve_store(session, store_api) + job = enqueue_for_enrollment( + session, enrollment_id, action, store_api=api, now=now + ) + if job.state == RetentionJobState.COMPLETED: + return job + return run_retention_job( + session, + job.job_id, + store_api=api, + now=now, + max_attempts=max_attempts, + ) diff --git a/src/research/runtime/__init__.py b/src/research/runtime/__init__.py new file mode 100644 index 0000000..0d1bd8f --- /dev/null +++ b/src/research/runtime/__init__.py @@ -0,0 +1,13 @@ +"""Research runtime: assignment, bootstrap and session lifecycle. + +Merged home for the enrollment-scoped assignment/exposure services, the +secret-free bootstrap manifest and capability issuance, and the research +session/agent-run state machine. The individual sub-packages keep their +original public surfaces: + +* :mod:`research.runtime.assignment` +* :mod:`research.runtime.bootstrap` +* :mod:`research.runtime.sessions` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/runtime/assignment/__init__.py b/src/research/runtime/assignment/__init__.py new file mode 100644 index 0000000..e10553d --- /dev/null +++ b/src/research/runtime/assignment/__init__.py @@ -0,0 +1,50 @@ +"""Enrollment-scoped assignment and exposure (Issue 05). + +Public surface: + +* :mod:`research.runtime.assignment.enums` - allocation/exposure outcomes and typed + reason codes. +* :mod:`research.runtime.assignment.models` - ``AssignmentV1`` and ``ExposureV1`` (two + separate immutable facts) plus typed results. +* :mod:`research.runtime.assignment.service` - server-authoritative, sticky, weighted + allocation over ``enrollment_id``. +* :mod:`research.runtime.assignment.exposure` - idempotent exposure receipts. +* :mod:`research.runtime.assignment.store` - persistence adapters taking a caller- + supplied SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ( + AllocationOutcome, + AssignmentReasonCode, + ExposureOutcome, + ExposureReasonCode, +) +from .exposure import record_exposure +from .models import ( + AssignmentIssue, + AssignmentResult, + AssignmentV1, + ExposureEnvironment, + ExposureIssue, + ExposureResult, + ExposureV1, +) +from .service import allocate + +__all__ = [ + "AllocationOutcome", + "AssignmentIssue", + "AssignmentReasonCode", + "AssignmentResult", + "AssignmentV1", + "ExposureEnvironment", + "ExposureIssue", + "ExposureOutcome", + "ExposureReasonCode", + "ExposureResult", + "ExposureV1", + "allocate", + "record_exposure", +] diff --git a/src/research/runtime/assignment/enums.py b/src/research/runtime/assignment/enums.py new file mode 100644 index 0000000..cc61e7a --- /dev/null +++ b/src/research/runtime/assignment/enums.py @@ -0,0 +1,77 @@ +"""Closed vocabularies for enrollment-scoped assignment and exposure (Issue 05). + +Every member is part of a persisted contract (assignment rows, exposure +receipts, API responses), so members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "AllocationOutcome", + "AssignmentReasonCode", + "ExposureOutcome", + "ExposureReasonCode", +] + + +class AllocationOutcome(str, Enum): + """Terminal result of an allocation attempt.""" + + CREATED = "CREATED" + EXISTING = "EXISTING" + INELIGIBLE = "INELIGIBLE" + INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE" + CONFLICT = "CONFLICT" + + +class AssignmentReasonCode(str, Enum): + """Stable machine-readable reason for an allocation outcome.""" + + OK = "OK" + NO_ENROLLMENT = "NO_ENROLLMENT" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + REVISION_NOT_PUBLISHED = "REVISION_NOT_PUBLISHED" + REVISION_MISMATCH = "REVISION_MISMATCH" + NO_CONDITIONS = "NO_CONDITIONS" + WEIGHTS_INVALID = "WEIGHTS_INVALID" + STICKY_EXISTING = "STICKY_EXISTING" + WEIGHTED_DRAW = "WEIGHTED_DRAW" + DETERMINISTIC_HASH = "DETERMINISTIC_HASH" + UNKNOWN_STRATEGY = "UNKNOWN_STRATEGY" + STRATIFIED_UNSUPPORTED = "STRATIFIED_UNSUPPORTED" + + +class ExposureOutcome(str, Enum): + """Outcome of an attempted runtime exposure. + + ``STARTED`` and ``SUCCEEDED`` count as an exposure; ``FAILED``, + ``RUNTIME_UNAVAILABLE`` and ``INCOMPATIBLE_ENVIRONMENT`` are recorded as + non-exposures with evidence so a launch failure is never counted as a + successful condition exposure. + """ + + STARTED = "STARTED" + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + RUNTIME_UNAVAILABLE = "RUNTIME_UNAVAILABLE" + INCOMPATIBLE_ENVIRONMENT = "INCOMPATIBLE_ENVIRONMENT" + + @property + def is_exposure(self) -> bool: + """Whether this outcome represents an actual condition exposure.""" + return self in (ExposureOutcome.STARTED, ExposureOutcome.SUCCEEDED) + + +class ExposureReasonCode(str, Enum): + """Stable machine-readable reason for an exposure receipt outcome.""" + + OK = "OK" + IDEMPOTENT_REPLAY = "IDEMPOTENT_REPLAY" + IDEMPOTENCY_KEY_REQUIRED = "IDEMPOTENCY_KEY_REQUIRED" + CONFLICT = "CONFLICT" + ASSIGNMENT_MISMATCH = "ASSIGNMENT_MISMATCH" + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" diff --git a/src/research/runtime/assignment/exposure.py b/src/research/runtime/assignment/exposure.py new file mode 100644 index 0000000..a8e9639 --- /dev/null +++ b/src/research/runtime/assignment/exposure.py @@ -0,0 +1,120 @@ +"""Idempotent exposure receipts. + +An exposure is recorded once an agent handshake actually starts (or a launch +fails). Receipts are idempotent by key: a replay preserves the first terminal +result and audits the retry, and a launch failure is stored as a non-exposure +with evidence. Recording an exposure never rewrites the assignment. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Callable, Optional + +from .enums import ExposureOutcome, ExposureReasonCode +from .models import ( + AssignmentV1, + ExposureEnvironment, + ExposureIssue, + ExposureResult, + ExposureV1, +) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _rejected( + code: ExposureReasonCode, message: str, field: str = "" +) -> ExposureResult: + return ExposureResult( + accepted=False, + reason=code, + issue=ExposureIssue(code=code, message=message, field=field), + ) + + +def record_exposure( + assignment: AssignmentV1, + environment: ExposureEnvironment, + outcome: ExposureOutcome, + idempotency_key: str, + *, + existing: Optional[ExposureV1] = None, + agent_release_id: Optional[str] = None, + artifact_digest: Optional[str] = None, + adapter_version: Optional[str] = None, + observed_configuration: Optional[dict] = None, + evidence_digest: Optional[str] = None, + now: Optional[datetime] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> ExposureResult: + """Record (or idempotently replay) one exposure receipt for an assignment.""" + timestamp = _now(now) + + # Operator kill switch: no exposure may be written while engaged. The check + # is injected so this service keeps no operations dependency. + if kill_switch_check is not None and kill_switch_check(): + return _rejected( + ExposureReasonCode.KILL_SWITCH_ENGAGED, + "an operator kill switch is engaged for this scope", + "kill_switch", + ) + + if assignment is None: + return _rejected( + ExposureReasonCode.ASSIGNMENT_MISMATCH, + "an assignment is required to record an exposure", + "assignment_id", + ) + + if not idempotency_key or not idempotency_key.strip(): + return _rejected( + ExposureReasonCode.IDEMPOTENCY_KEY_REQUIRED, + "an idempotency key is required for an exposure receipt", + "idempotency_key", + ) + + if existing is not None: + if existing.idempotency_key == idempotency_key: + return ExposureResult( + accepted=True, + exposure=existing, + reused=True, + is_exposure=existing.is_exposure, + reason=ExposureReasonCode.IDEMPOTENT_REPLAY, + audit=[ + "idempotent replay; first terminal outcome preserved " + f"({existing.outcome.value})" + ], + ) + return _rejected( + ExposureReasonCode.CONFLICT, + "an exposure is already recorded for this assignment under a different key", + "idempotency_key", + ) + + exposure = ExposureV1( + exposure_id=uuid.uuid4(), + assignment_id=assignment.assignment_id, + study_revision_id=assignment.study_revision_id, + environment=environment, + agent_release_id=agent_release_id or "", + artifact_digest=artifact_digest, + adapter_version=adapter_version, + observed_configuration=dict(observed_configuration or {}), + started_at=timestamp, + outcome=outcome, + evidence_digest=evidence_digest, + idempotency_key=idempotency_key, + created_at=timestamp, + ) + return ExposureResult( + accepted=True, + exposure=exposure, + reused=False, + is_exposure=exposure.is_exposure, + reason=ExposureReasonCode.OK, + ) diff --git a/src/research/runtime/assignment/models.py b/src/research/runtime/assignment/models.py new file mode 100644 index 0000000..88b7af8 --- /dev/null +++ b/src/research/runtime/assignment/models.py @@ -0,0 +1,126 @@ +"""Pydantic v2 contracts for enrollment-scoped assignment and exposure. + +Assignment and exposure are **separate immutable facts**: + +* :class:`AssignmentV1` is the server-authoritative, sticky condition allocated + for one ``(enrollment_id, study_revision_id)``. It is never re-randomized for + a revision. +* :class:`ExposureV1` records what the runtime actually started, with its own + id and timestamp; it never rewrites the assignment. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + AllocationOutcome, + AssignmentReasonCode, + ExposureOutcome, + ExposureReasonCode, +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class AssignmentV1(BaseModel): + """The immutable condition assigned to one enrollment/revision.""" + + model_config = _FROZEN + + assignment_id: UUID + enrollment_id: UUID + study_revision_id: UUID + condition_id: str + strategy: str + randomization_epoch: int = 0 + assigned_at: datetime + protocol_digest: str + + +class ExposureEnvironment(BaseModel): + """Host environment an exposure was attempted in.""" + + model_config = _BASE + + os: str + arch: str + ide_build: Optional[str] = None + plugin_version: Optional[str] = None + host_kind: Optional[str] = None + + +class ExposureV1(BaseModel): + """The durable record of one attempted runtime exposure.""" + + model_config = _FROZEN + + exposure_id: UUID + assignment_id: UUID + study_revision_id: UUID + environment: ExposureEnvironment + agent_release_id: str + artifact_digest: Optional[str] = None + adapter_version: Optional[str] = None + observed_configuration: dict[str, Any] = Field(default_factory=dict) + started_at: datetime + outcome: ExposureOutcome + evidence_digest: Optional[str] = None + idempotency_key: str + created_at: datetime + + @property + def is_exposure(self) -> bool: + """Whether this receipt counts as an actual condition exposure.""" + return self.outcome.is_exposure + + +class AssignmentIssue(BaseModel): + """One typed assignment/exposure rejection reason.""" + + model_config = _BASE + + code: AssignmentReasonCode + message: str + field: str = "" + + +class AssignmentResult(BaseModel): + """Typed outcome of an allocation attempt.""" + + model_config = _BASE + + outcome: AllocationOutcome + assignment: Optional[AssignmentV1] = None + created: bool = False + reason: AssignmentReasonCode = AssignmentReasonCode.OK + issue: Optional[AssignmentIssue] = None + + +class ExposureIssue(BaseModel): + """Typed exposure rejection reason.""" + + model_config = _BASE + + code: ExposureReasonCode + message: str + field: str = "" + + +class ExposureResult(BaseModel): + """Typed outcome of recording an exposure receipt.""" + + model_config = _BASE + + accepted: bool + exposure: Optional[ExposureV1] = None + reused: bool = False + is_exposure: bool = False + reason: ExposureReasonCode = ExposureReasonCode.OK + audit: list[str] = Field(default_factory=list) + issue: Optional[ExposureIssue] = None diff --git a/src/research/runtime/assignment/service.py b/src/research/runtime/assignment/service.py new file mode 100644 index 0000000..7f51172 --- /dev/null +++ b/src/research/runtime/assignment/service.py @@ -0,0 +1,220 @@ +"""Pure server-authoritative allocation over enrollment-scoped units. + +The unit of assignment is ``enrollment_id`` (never account, device, task, or +process). Assignments are always sticky: when one already exists for the same +``(enrollment_id, study_revision_id)`` it is returned unchanged and never +re-randomized, and a changed weight or a later revision never rebuckets an +existing enrollment. There is no runtime switch. Reallocation, if it is ever +required, must be defined by a successor revision rather than by mutating an +existing assignment. Ambiguous eligibility fails closed with a typed reason +rather than a default arm. +""" + +from __future__ import annotations + +import random +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from research.canonical import canonical_hash +from research.participants.enums import EnrollmentStatus +from research.study.protocol.enums import AssignmentStrategy, RevisionStatus +from research.study.protocol.models import StudyProtocolV1 +from research.study.protocol.validation import ( + ProtocolValidationError, + normalized_condition_weights, +) + +from .enums import AllocationOutcome, AssignmentReasonCode +from .models import AssignmentIssue, AssignmentResult, AssignmentV1 + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from research.study.protocol.publication import StudyRevision + + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: AssignmentReasonCode, message: str, field: str = "") -> AssignmentIssue: + return AssignmentIssue(code=code, message=message, field=field) + + +def _blocked( + outcome: AllocationOutcome, code: AssignmentReasonCode, message: str, field: str = "" +) -> AssignmentResult: + return AssignmentResult( + outcome=outcome, reason=code, issue=_issue(code, message, field) + ) + + +def _protocol(revision: StudyRevision) -> StudyProtocolV1: + return StudyProtocolV1.model_validate(revision.protocol_json) + + +def _weighted_condition( + weights: dict[str, float], rng: random.Random +) -> str: + draw = rng.random() + cumulative = 0.0 + last = "" + for condition_id in sorted(weights): + last = condition_id + cumulative += weights[condition_id] + if draw < cumulative: + return condition_id + return last + + +def _deterministic_condition( + weights: dict[str, float], + *, + enrollment_id: uuid.UUID, + revision_id: uuid.UUID, + randomization_epoch: int, +) -> str: + seed = canonical_hash( + { + "enrollment_id": str(enrollment_id), + "revision_id": str(revision_id), + "randomization_epoch": randomization_epoch, + } + ) + draw = int(seed, 16) / float(1 << 256) + cumulative = 0.0 + last = "" + for condition_id in sorted(weights): + last = condition_id + cumulative += weights[condition_id] + if draw < cumulative: + return condition_id + return last + + +def allocate( + enrollment: Enrollment, + revision: StudyRevision, + existing: Optional[AssignmentV1] = None, + rng: Optional[random.Random] = None, + now: Optional[datetime] = None, +) -> AssignmentResult: + """Allocate (or return) the sticky condition for one enrollment/revision.""" + timestamp = _now(now) + + if enrollment is None: + return _blocked( + AllocationOutcome.INELIGIBLE, + AssignmentReasonCode.NO_ENROLLMENT, + "an enrollment is required for allocation", + "enrollment_id", + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _blocked( + AllocationOutcome.INELIGIBLE, + AssignmentReasonCode.ENROLLMENT_NOT_ACTIVE, + f"enrollment is {enrollment.status.value}", + "status", + ) + + if revision is None or revision.revision_id != enrollment.study_revision_id: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.REVISION_MISMATCH, + "enrollment is not bound to the requested revision", + "study_revision_id", + ) + + if revision.status != RevisionStatus.PUBLISHED: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.REVISION_NOT_PUBLISHED, + f"revision is {revision.status.value}; only PUBLISHED revisions allocate", + "status", + ) + + if existing is not None: + if ( + existing.enrollment_id == enrollment.enrollment_id + and existing.study_revision_id == revision.revision_id + ): + return AssignmentResult( + outcome=AllocationOutcome.EXISTING, + assignment=existing, + created=False, + reason=AssignmentReasonCode.STICKY_EXISTING, + ) + return _blocked( + AllocationOutcome.CONFLICT, + AssignmentReasonCode.REVISION_MISMATCH, + "an assignment already exists for a different enrollment/revision", + "study_revision_id", + ) + + protocol = _protocol(revision) + if not protocol.conditions: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.NO_CONDITIONS, + "the revision declares no conditions", + "conditions", + ) + + try: + weights = normalized_condition_weights(protocol) + except ProtocolValidationError: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.WEIGHTS_INVALID, + "condition weights are not normalizable", + "conditions", + ) + + strategy = protocol.assignment.strategy + randomization_epoch = 0 + + if strategy == AssignmentStrategy.WEIGHTED_RANDOM.value: + condition_id = _weighted_condition(weights, rng or random.Random()) + reason = AssignmentReasonCode.WEIGHTED_DRAW + elif strategy == AssignmentStrategy.DETERMINISTIC_HASH.value: + condition_id = _deterministic_condition( + weights, + enrollment_id=enrollment.enrollment_id, + revision_id=revision.revision_id, + randomization_epoch=randomization_epoch, + ) + reason = AssignmentReasonCode.DETERMINISTIC_HASH + elif strategy == AssignmentStrategy.STRATIFIED.value: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.STRATIFIED_UNSUPPORTED, + "stratified allocation requires stratum values that are not available", + "assignment.strategy", + ) + else: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.UNKNOWN_STRATEGY, + f"unknown assignment strategy {strategy!r}", + "assignment.strategy", + ) + + assignment = AssignmentV1( + assignment_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_revision_id=revision.revision_id, + condition_id=condition_id, + strategy=strategy, + randomization_epoch=randomization_epoch, + assigned_at=timestamp, + protocol_digest=revision.protocol_digest, + ) + return AssignmentResult( + outcome=AllocationOutcome.CREATED, + assignment=assignment, + created=True, + reason=reason, + ) diff --git a/src/research/runtime/assignment/store.py b/src/research/runtime/assignment/store.py new file mode 100644 index 0000000..e5246c3 --- /dev/null +++ b/src/research/runtime/assignment/store.py @@ -0,0 +1,170 @@ +"""CRUD-style persistence helpers for assignment and exposure. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. Assignments are +immutable facts; the unique ``(enrollment_id, study_revision_id)`` constraint is +the concurrency guard, so a concurrent first-bootstrap race yields one row and +the loser re-reads it. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import AssignmentV1, ExposureV1 + +from database.research_schemas import StudyAssignment + +from .enums import ExposureOutcome + + +def create_assignment( + session: Session, assignment: AssignmentV1, *, commit: bool = True +) -> StudyAssignment: + """Insert one immutable assignment row, or return the existing winner. + + The unique ``(enrollment_id, study_revision_id)`` constraint is the + concurrency guard: a savepoint around the insert means a losing concurrent + create rolls back only its own insert and returns the winning row, so the + caller always uses the authoritative sticky assignment. ``commit=False`` lets + bootstrap own one unit of work. + """ + row = StudyAssignment( + assignment_id=assignment.assignment_id, + enrollment_id=assignment.enrollment_id, + study_revision_id=assignment.study_revision_id, + condition_id=assignment.condition_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + protocol_digest=assignment.protocol_digest, + assigned_at=assignment.assigned_at, + ) + try: + with session.begin_nested(): + session.add(row) + except IntegrityError: + existing = get_assignment_for_enrollment_revision( + session, assignment.enrollment_id, assignment.study_revision_id + ) + if existing is not None: + return existing + raise + if commit: + session.commit() + else: + session.flush() + session.refresh(row) + return row + + +def get_assignment( + session: Session, assignment_id: uuid.UUID +) -> Optional[StudyAssignment]: + """Fetch an assignment by id, or ``None``.""" + return session.get(StudyAssignment, assignment_id) + + +def get_assignment_for_enrollment_revision( + session: Session, enrollment_id: uuid.UUID, study_revision_id: uuid.UUID +) -> Optional[StudyAssignment]: + """Fetch the unique sticky assignment for ``(enrollment, revision)``.""" + statement = select(StudyAssignment).where( + StudyAssignment.enrollment_id == enrollment_id, + StudyAssignment.study_revision_id == study_revision_id, + ) + return session.execute(statement).scalars().first() + + +def list_assignments( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[StudyAssignment]: + """List assignments, optionally for one enrollment.""" + statement = select(StudyAssignment) + if enrollment_id is not None: + statement = statement.where(StudyAssignment.enrollment_id == enrollment_id) + statement = statement.order_by(StudyAssignment.assigned_at.asc()) + return list(session.execute(statement).scalars().all()) + + +def insert_exposure(session: Session, exposure: ExposureV1) -> Any: + """Reject the removed condition-exposure persistence contract.""" + raise NotImplementedError("condition exposure persistence was removed") + + +def get_exposure_by_idempotency_key( + session: Session, idempotency_key: str +) -> Any: + """Reject the removed condition-exposure lookup contract.""" + raise NotImplementedError("condition exposure persistence was removed") + + +def list_exposures( + session: Session, assignment_id: Optional[uuid.UUID] = None +) -> Sequence[Any]: + """Reject the removed condition-exposure listing contract.""" + raise NotImplementedError("condition exposure persistence was removed") + + +def row_to_assignment(row: StudyAssignment) -> AssignmentV1: + """Rehydrate an assignment row into the domain model.""" + from .models import AssignmentV1 as AssignmentV1Model + + return AssignmentV1Model( + assignment_id=row.assignment_id, + enrollment_id=row.enrollment_id, + study_revision_id=row.study_revision_id, + condition_id=row.condition_id, + strategy=row.strategy, + randomization_epoch=row.randomization_epoch, + assigned_at=row.assigned_at, + protocol_digest=row.protocol_digest, + ) + + +def row_to_exposure(row: Any) -> ExposureV1: + """Rehydrate an exposure row into the domain model.""" + from .models import ExposureEnvironment + from .models import ExposureV1 as ExposureV1Model + + return ExposureV1Model( + exposure_id=row.exposure_id, + assignment_id=row.assignment_id, + study_revision_id=row.study_revision_id, + environment=ExposureEnvironment.model_validate(row.environment_json or {}), + agent_release_id=row.agent_release_id, + artifact_digest=row.artifact_digest, + adapter_version=row.adapter_version, + observed_configuration=row.observed_configuration or {}, + started_at=row.started_at, + outcome=ExposureOutcome(row.outcome), + evidence_digest=row.evidence_digest, + idempotency_key=row.idempotency_key, + created_at=row.created_at, + ) + + +def exposure_summary(row: Any) -> dict[str, Any]: + """Return a compact exposure summary safe for responses.""" + started_at = row.started_at + return { + "exposure_id": str(row.exposure_id), + "assignment_id": str(row.assignment_id), + "study_revision_id": str(row.study_revision_id), + "agent_release_id": row.agent_release_id, + "artifact_digest": row.artifact_digest, + "outcome": row.outcome, + "started_at": ( + started_at.isoformat() if isinstance(started_at, datetime) else None + ), + "evidence_digest": row.evidence_digest, + "idempotency_key": row.idempotency_key, + } diff --git a/src/research/runtime/bootstrap/__init__.py b/src/research/runtime/bootstrap/__init__.py new file mode 100644 index 0000000..af070b9 --- /dev/null +++ b/src/research/runtime/bootstrap/__init__.py @@ -0,0 +1,95 @@ +"""Bootstrap manifest and session capability (Issue 05). + +Public surface: + +* :mod:`research.runtime.bootstrap.models` - ``BootstrapManifestV1`` and its sub-objects, + ``SessionCapability``, typed reason codes and results. +* :mod:`research.runtime.bootstrap.signer` - dependency-free manifest digest + HMAC + signature and verification. +* :mod:`research.runtime.bootstrap.capability` - scoped, short-lived capability issuance + and server-side verification. +* :mod:`research.runtime.bootstrap.service` - :func:`compose_bootstrap`, which validates + eligibility/compatibility before projecting and signing a secret-free manifest. +* :mod:`research.runtime.bootstrap.store` - persistence adapters taking a caller-supplied + SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .capability import ( + DEFAULT_AUDIENCE, + DEFAULT_SCOPE, + compute_capability_signature, + issue_capability, + verify_capability, +) +from .models import ( + BootstrapAgentRelease, + BootstrapAssignment, + BootstrapCompatibility, + BootstrapIssue, + BootstrapManifestV1, + BootstrapOutcome, + BootstrapPolicies, + BootstrapPrivacyPolicy, + BootstrapReasonCode, + BootstrapResult, + BootstrapSessionPolicy, + BootstrapTelemetryPolicy, + CapabilityReasonCode, + CapabilityVerification, + ManifestReasonCode, + ManifestSignature, + ManifestVerification, + ResearchSessionRef, + SessionCapability, +) +from .service import ( + BootstrapSigningContext, + EphemeralSessionFactory, + SessionFactory, + compose_bootstrap, +) +from .signer import ( + canonical_manifest_bytes, + compute_manifest_digest, + manifest_digest_payload, + sign_manifest, + verify_manifest, +) + +__all__ = [ + "DEFAULT_AUDIENCE", + "DEFAULT_SCOPE", + "BootstrapAgentRelease", + "BootstrapAssignment", + "BootstrapCompatibility", + "BootstrapIssue", + "BootstrapManifestV1", + "BootstrapOutcome", + "BootstrapPolicies", + "BootstrapPrivacyPolicy", + "BootstrapReasonCode", + "BootstrapResult", + "BootstrapSessionPolicy", + "BootstrapSigningContext", + "BootstrapTelemetryPolicy", + "CapabilityReasonCode", + "CapabilityVerification", + "EphemeralSessionFactory", + "ManifestReasonCode", + "ManifestSignature", + "ManifestVerification", + "ResearchSessionRef", + "SessionCapability", + "SessionFactory", + "canonical_manifest_bytes", + "compose_bootstrap", + "compute_capability_signature", + "compute_manifest_digest", + "issue_capability", + "manifest_digest_payload", + "sign_manifest", + "verify_capability", + "verify_manifest", +] diff --git a/src/research/runtime/bootstrap/capability.py b/src/research/runtime/bootstrap/capability.py new file mode 100644 index 0000000..11cfce9 --- /dev/null +++ b/src/research/runtime/bootstrap/capability.py @@ -0,0 +1,204 @@ +"""Scoped, short-lived session capabilities. + +A capability is signed with HMAC-SHA256 over its canonical mapping (excluding +its own signature) and is evaluated server-side for expiry, audience, scope and +revocation epoch. It never carries a provider or account secret. +""" + +from __future__ import annotations + +import hashlib +import hmac +import uuid +from datetime import datetime, timedelta, timezone +from typing import Optional + +from research.canonical import canonical_hash + +from .models import ( + CapabilityReasonCode, + CapabilityVerification, + SessionCapability, +) + +__all__ = [ + "capability_payload", + "compute_capability_signature", + "issue_capability", + "verify_capability", +] + +DEFAULT_AUDIENCE = "research-runtime" +DEFAULT_SCOPE = ("telemetry:write", "session:heartbeat", "session:close") + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def capability_payload(capability: SessionCapability) -> dict: + """Return the canonical mapping covered by the capability signature.""" + data = capability.model_dump(mode="json") + data.pop("signature", None) + return data + + +def _signature_for(capability: SessionCapability, secret: str) -> str: + digest = canonical_hash(capability_payload(capability)) + return hmac.new( + secret.encode("utf-8"), digest.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + +def compute_capability_signature(capability: SessionCapability, secret: str) -> str: + """Return the expected HMAC signature for a capability.""" + return _signature_for(capability, secret) + + +def issue_capability( + audience: str, + scope: list[str], + ttl_seconds: int, + revocation_epoch: int, + secret: str, + now: Optional[datetime] = None, + *, + enrollment_id: uuid.UUID, + research_session_id: uuid.UUID, + revision_id: uuid.UUID, +) -> SessionCapability: + """Issue a signed, short-lived capability bound to a subject. + + ``enrollment_id``, ``research_session_id`` and ``revision_id`` are required + and covered by the HMAC, so the capability is only usable for the exact + enrollment/session/revision it was issued for. There is no default secret: + issuing without a configured signing secret is refused. + """ + if ttl_seconds <= 0: + raise ValueError("ttl_seconds must be positive") + if not secret or not secret.strip(): + raise ValueError("a non-empty signing secret is required to issue a capability") + + issued_at = _now(now) + capability = SessionCapability( + capability_id=uuid.uuid4(), + audience=audience, + scope=sorted(set(scope)), + issued_at=issued_at, + expires_at=issued_at + timedelta(seconds=ttl_seconds), + revocation_epoch=revocation_epoch, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + revision_id=revision_id, + signature="", + ) + return capability.model_copy(update={"signature": _signature_for(capability, secret)}) + + +def verify_capability( + capability: SessionCapability, + secret: str, + expected_audience: str, + expected_scope: list[str], + now: Optional[datetime] = None, + current_revocation_epoch: Optional[int] = None, + *, + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_revision_id: Optional[uuid.UUID] = None, +) -> CapabilityVerification: + """Verify a capability server-side, returning a typed reason. + + The subject arguments are optional so a stateless signature/expiry check can + still be performed without a resolved resource, but every authorization use + passes them so a valid capability for another subject can never be replayed. + """ + if not secret or not secret.strip(): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SIGNING_SECRET_MISSING, + message="no signing secret is configured; refusing to verify capabilities", + ) + + if not capability.signature or not hmac.compare_digest( + capability.signature, _signature_for(capability, secret) + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SIGNATURE_MISMATCH, + message="capability signature does not match", + ) + + if ( + expected_enrollment_id is not None + and capability.enrollment_id != expected_enrollment_id + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SUBJECT_MISMATCH, + message="capability belongs to a different enrollment", + ) + + if ( + expected_research_session_id is not None + and capability.research_session_id != expected_research_session_id + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SESSION_MISMATCH, + message="capability belongs to a different research session", + ) + + if ( + expected_revision_id is not None + and capability.revision_id != expected_revision_id + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.REVISION_MISMATCH, + message="capability is bound to a different revision", + ) + + if capability.audience != expected_audience: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.WRONG_AUDIENCE, + message=( + f"capability audience {capability.audience!r} does not match " + f"{expected_audience!r}" + ), + ) + + missing = sorted(set(expected_scope) - set(capability.scope)) + if missing: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SCOPE_MISSING, + message=f"capability is missing required scope {missing}", + ) + + timestamp = _now(now) + if timestamp < capability.issued_at: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.NOT_YET_VALID, + message="capability is not valid yet", + ) + if timestamp >= capability.expires_at: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.EXPIRED, + message="capability has expired", + ) + + if ( + current_revocation_epoch is not None + and capability.revocation_epoch != current_revocation_epoch + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.REVOKED, + message="capability revocation epoch is stale", + ) + + return CapabilityVerification(ok=True, reason=CapabilityReasonCode.OK) diff --git a/src/research/runtime/bootstrap/models.py b/src/research/runtime/bootstrap/models.py new file mode 100644 index 0000000..2ddf5c6 --- /dev/null +++ b/src/research/runtime/bootstrap/models.py @@ -0,0 +1,297 @@ +"""Pydantic v2 contracts for the bootstrap manifest and session capability. + +The manifest is a signed, short-lived, secret-free projection. It contains only +the pinned revision reference, the sticky assignment, the pinned agent release, +the policy set, an optional compatibility receipt reference, and a scoped +session capability. It never contains account identity, provider credentials, +raw consent documents, or arbitrary launch commands; ``null`` optional policy +values mean inherited/disabled, and unavailable capabilities carry a typed +state and reason. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from enum import Enum +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class BootstrapOutcome(str, Enum): + """Terminal result of composing a bootstrap manifest.""" + + ISSUED = "ISSUED" + BLOCKED = "BLOCKED" + + +class BootstrapReasonCode(str, Enum): + """Stable machine-readable reason a bootstrap was blocked (or issued).""" + + OK = "OK" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + STUDY_NOT_OPEN = "STUDY_NOT_OPEN" + STUDY_CLOSED = "STUDY_CLOSED" + REVISION_NOT_PUBLISHED = "REVISION_NOT_PUBLISHED" + REVISION_MISMATCH = "REVISION_MISMATCH" + ASSIGNMENT_MISMATCH = "ASSIGNMENT_MISMATCH" + RELEASE_NOT_QUALIFIED = "RELEASE_NOT_QUALIFIED" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" + ARTIFACT_UNAVAILABLE = "ARTIFACT_UNAVAILABLE" + ARTIFACT_MISMATCH = "ARTIFACT_MISMATCH" + COMPATIBILITY_MISSING = "COMPATIBILITY_MISSING" + INCOMPATIBLE_ENVIRONMENT = "INCOMPATIBLE_ENVIRONMENT" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + # No signing secret is configured, so no capability/manifest may be issued. + SIGNING_SECRET_MISSING = "SIGNING_SECRET_MISSING" + # Operator kill switch: no new session may be bootstrapped while engaged. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + + +class ManifestReasonCode(str, Enum): + """Stable machine-readable manifest verification reason.""" + + OK = "OK" + MISSING_SIGNATURE = "MISSING_SIGNATURE" + MANIFEST_DIGEST_MISMATCH = "MANIFEST_DIGEST_MISMATCH" + MANIFEST_SIGNATURE_MISMATCH = "MANIFEST_SIGNATURE_MISMATCH" + + +class CapabilityReasonCode(str, Enum): + """Stable machine-readable capability verification reason.""" + + OK = "OK" + SIGNATURE_MISMATCH = "SIGNATURE_MISMATCH" + SIGNING_SECRET_MISSING = "SIGNING_SECRET_MISSING" + WRONG_AUDIENCE = "WRONG_AUDIENCE" + SCOPE_MISSING = "SCOPE_MISSING" + NOT_YET_VALID = "NOT_YET_VALID" + EXPIRED = "EXPIRED" + REVOKED = "REVOKED" + # The capability is cryptographically valid but belongs to a different + # subject than the resource it is being presented for. + SUBJECT_MISMATCH = "SUBJECT_MISMATCH" + SESSION_MISMATCH = "SESSION_MISMATCH" + REVISION_MISMATCH = "REVISION_MISMATCH" + + +class ResearchSessionRef(BaseModel): + """Opaque research session descriptor (Issue 07 owns its lifecycle).""" + + model_config = _BASE + + research_session_id: UUID + opened_at: datetime + + +class SessionCapability(BaseModel): + """Short-lived, scoped, signed session capability bound to its subject. + + The capability is only meaningful for one ``(enrollment, research session, + revision)`` triple. It carries that subject explicitly and the signed + payload covers it, so a capability issued for one participant cannot be + replayed against another participant's enrollment, session, or revision. + """ + + model_config = _FROZEN + + capability_id: UUID + audience: str + scope: list[str] = Field(default_factory=list) + issued_at: datetime + expires_at: datetime + revocation_epoch: int = 0 + # The subject the capability is bound to. All three are required so a + # capability can never be issued without an explicit binding. + enrollment_id: UUID + research_session_id: UUID + revision_id: UUID + signature: str = "" + + +class BootstrapAssignment(BaseModel): + """The assignment projection embedded in a manifest.""" + + model_config = _FROZEN + + assignment_id: UUID + condition_id: str + strategy: str + randomization_epoch: int = 0 + protocol_digest: str + + +class BootstrapAgentRelease(BaseModel): + """The pinned agent release projection embedded in a manifest. + + ``distribution_mode`` selects how the agent reaches the participant: + ``PACKAGED`` (default) pins ``artifact_digest``; ``BYOA_EXTERNAL`` pins a + participant-installed command/package identity and leaves + ``artifact_digest`` empty. + """ + + model_config = _FROZEN + + agent_id: str + release_id: str + artifact_digest: str = "" + adapter_version: Optional[str] = None + distribution_mode: str = "PACKAGED" + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + agent_package: Optional[str] = None + + +class BootstrapAgentProfile(BaseModel): + """The non-secret provider/model identity of a condition's agent profile. + + This is a projection of the server-side ``AgentProfile`` the condition + references. It deliberately carries no credential and not even the + ``api_key_ref`` env-var *name*: the runtime relays inference through the + backend, which resolves and injects the key server-side, so the secret and + its reference never leave the server. + """ + + model_config = _FROZEN + + profile_id: UUID + name: str + framework_version: str + model: str + base_url: Optional[str] = None + temperature: Optional[float] = None + + +class BootstrapTelemetryPolicy(BaseModel): + """Telemetry field-class allowance projected into a manifest.""" + + model_config = _FROZEN + + allowed_field_classes: list[str] = Field(default_factory=list) + content_capture: bool = False + + +class BootstrapPrivacyPolicy(BaseModel): + """Retention policy projected into a manifest.""" + + model_config = _FROZEN + + retention_action: str + retention_days: Optional[int] = None + + +class BootstrapSessionPolicy(BaseModel): + """Session timing policy projected into a manifest.""" + + model_config = _FROZEN + + idle_timeout_seconds: Optional[int] = None + resume_grace_seconds: Optional[int] = None + heartbeat_seconds: Optional[int] = None + + +class BootstrapPolicies(BaseModel): + """The policy set; ``null`` members mean inherited/disabled.""" + + model_config = _FROZEN + + telemetry_policy: Optional[BootstrapTelemetryPolicy] = None + privacy_policy: Optional[BootstrapPrivacyPolicy] = None + session_policy: Optional[BootstrapSessionPolicy] = None + consent_policy_digest: Optional[str] = None + + +class BootstrapCompatibility(BaseModel): + """Compatibility projection with an explicit coverage state and reason.""" + + model_config = _FROZEN + + receipt_ref: Optional[str] = None + state: CoverageState = CoverageState.UNKNOWN + reason: Optional[str] = None + + +class BootstrapManifestV1(BaseModel): + """Signed, short-lived, secret-free launch contract (schema version 1).""" + + model_config = _FROZEN + + manifest_version: str = "1" + manifest_digest: str = "" + signature: str = "" + generated_at: datetime + study_id: UUID + revision_id: UUID + revision_digest: str + enrollment_id: UUID + research_session: ResearchSessionRef + assignment: BootstrapAssignment + agent_release: BootstrapAgentRelease + # Optional non-secret provider/model identity of the condition's linked + # agent profile. Absent (null) for protocols whose conditions predate the + # profile link or that pin a release without a profile. + agent_profile: Optional[BootstrapAgentProfile] = None + policies: BootstrapPolicies + compatibility_receipt_ref: Optional[str] = None + compatibility: BootstrapCompatibility = Field(default_factory=BootstrapCompatibility) + session_capability: SessionCapability + + +class ManifestSignature(BaseModel): + """Content digest and HMAC signature for a manifest.""" + + model_config = _BASE + + digest: str + signature: str + + +class ManifestVerification(BaseModel): + """Typed result of verifying a manifest's digest and signature.""" + + model_config = _BASE + + ok: bool + reason: ManifestReasonCode = ManifestReasonCode.OK + expected_digest: str = "" + actual_digest: str = "" + message: Optional[str] = None + + +class CapabilityVerification(BaseModel): + """Typed result of server-side capability verification.""" + + model_config = _BASE + + ok: bool + reason: CapabilityReasonCode = CapabilityReasonCode.OK + message: Optional[str] = None + + +class BootstrapIssue(BaseModel): + """One typed bootstrap block reason.""" + + model_config = _BASE + + code: BootstrapReasonCode + message: str + field: str = "" + + +class BootstrapResult(BaseModel): + """Typed outcome of composing a bootstrap manifest.""" + + model_config = _BASE + + outcome: BootstrapOutcome + reason: BootstrapReasonCode = BootstrapReasonCode.OK + manifest: Optional[BootstrapManifestV1] = None + issue: Optional[BootstrapIssue] = None diff --git a/src/research/runtime/bootstrap/service.py b/src/research/runtime/bootstrap/service.py new file mode 100644 index 0000000..4715c45 --- /dev/null +++ b/src/research/runtime/bootstrap/service.py @@ -0,0 +1,513 @@ +"""Bootstrap composition: validate everything, then project and sign. + +Nothing is issued until the enrollment is active with consent, the study window +is open, the revision is published, the assignment matches, and the pinned agent +release is qualified and resolvable for the participant platform. A revision that +declares required capabilities additionally needs a COMPATIBLE compatibility +result; a revision that declares none may proceed without a receipt. Every +failure is a typed block; the service never fails open and never invents a +default arm or fallback artifact. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Callable, Optional, Protocol + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.compatibility.enums import CompatibilityDecision +from research.participants.enums import EnrollmentStatus +from research.study.protocol.enums import ( + ReleaseResolutionStatus, + RevisionStatus, + TelemetryFieldClass, +) +from research.study.protocol.models import FixedSchedule, RollingSchedule, StudyProtocolV1 +from research.telemetry.enums import CoverageState + +from .capability import issue_capability +from .models import ( + BootstrapAgentProfile, + BootstrapAgentRelease, + BootstrapAssignment, + BootstrapCompatibility, + BootstrapIssue, + BootstrapManifestV1, + BootstrapOutcome, + BootstrapPolicies, + BootstrapPrivacyPolicy, + BootstrapReasonCode, + BootstrapResult, + BootstrapSessionPolicy, + BootstrapTelemetryPolicy, + ResearchSessionRef, +) +from .signer import sign_manifest + +if TYPE_CHECKING: + from research.compatibility.models import CompatibilityResult + from research.participants.models import Enrollment + from research.runtime.assignment.models import AssignmentV1 + from research.study.agents.models import AgentReleaseV1 + from research.study.protocol.publication import StudyRevision + from research.study.protocol.validation import ReleaseResolver + +__all__ = [ + "BootstrapSigningContext", + "EphemeralSessionFactory", + "SessionFactory", + "compose_bootstrap", +] + +_BASE = ConfigDict(extra="forbid") + + +class SessionFactory(Protocol): + """Creates (or reuses) a research session for an enrollment.""" + + def create_for_enrollment( + self, + enrollment: Enrollment, + revision: StudyRevision, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: # pragma: no cover - structural protocol + ... + + +class EphemeralSessionFactory: + """Test-only factory: returns a fresh session ref and persists nothing. + + This is **not** a production default. The production path always injects a + database-backed factory (:class:`backend.routers.research.bootstrap._PersistentSessionFactory`), + so a manifest's ``research_session_id`` resolves through the session store. + A build without a database passes this explicitly (tests); ``compose_bootstrap`` + has no default factory and never falls back to an in-memory one. + """ + + def create_for_enrollment( + self, + enrollment: Enrollment, + revision: StudyRevision, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: + return ResearchSessionRef(research_session_id=uuid.uuid4(), opened_at=now) + + +class BootstrapSigningContext(BaseModel): + """Injected signing parameters for manifest and capability issuance.""" + + model_config = _BASE + + secret: str + capability_ttl_seconds: int = 900 + audience: str = "research-runtime" + scope: list[str] = Field( + default_factory=lambda: [ + "telemetry:write", + "session:heartbeat", + "session:close", + ] + ) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _blocked( + code: BootstrapReasonCode, message: str, field: str = "" +) -> BootstrapResult: + return BootstrapResult( + outcome=BootstrapOutcome.BLOCKED, + reason=code, + issue=BootstrapIssue(code=code, message=message, field=field), + ) + + +def _schedule_issue( + protocol: StudyProtocolV1, enrollment: Enrollment, now: datetime +) -> Optional[BootstrapIssue]: + schedule = protocol.schedule + if isinstance(schedule, FixedSchedule): + if now < schedule.start_at: + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_NOT_OPEN, + message="the study has not opened yet", + field="schedule.start_at", + ) + if schedule.end_at is not None and now > schedule.end_at: + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_CLOSED, + message="the study window has closed", + field="schedule.end_at", + ) + return None + if isinstance(schedule, RollingSchedule): + duration = schedule.duration_seconds + if duration is None or duration <= 0: + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_CLOSED, + message="rolling duration is not usable", + field="schedule.duration_seconds", + ) + if now > enrollment.enrolled_at + timedelta(seconds=duration): + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_CLOSED, + message="the rolling enrollment window has closed", + field="schedule.duration_seconds", + ) + return None + + +def compose_bootstrap( + enrollment: Enrollment, + revision: StudyRevision, + assignment: AssignmentV1, + release: AgentReleaseV1, + compatibility_ref: Optional[str], + session_factory: SessionFactory, + signer: BootstrapSigningContext, + now: Optional[datetime] = None, + *, + compatibility_result: Optional[CompatibilityResult] = None, + platform: Optional[tuple[str, str]] = None, + release_resolver: Optional[ReleaseResolver] = None, + agent_profile: Optional[BootstrapAgentProfile] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, + context_id: str = "", +) -> BootstrapResult: + """Compose a signed, short-lived, secret-free bootstrap manifest.""" + timestamp = _now(now) + + # Operator kill switch (Issue 13): when engaged, no new session is issued. + # The check is injected so this service keeps no operations dependency. + if kill_switch_check is not None and kill_switch_check(): + return _blocked( + BootstrapReasonCode.KILL_SWITCH_ENGAGED, + "an operator kill switch is engaged for this scope", + "kill_switch", + ) + + # No signing secret is configured: never fall back to a hardcoded default, + # because a predictable secret would let anyone mint a capability. + if signer is None or not signer.secret or not signer.secret.strip(): + return _blocked( + BootstrapReasonCode.SIGNING_SECRET_MISSING, + "no bootstrap signing secret is configured; refusing to issue a manifest", + "signer.secret", + ) + + if enrollment is None: + return _blocked( + BootstrapReasonCode.ENROLLMENT_NOT_ACTIVE, + "an enrollment is required", + "enrollment_id", + ) + if enrollment.status != EnrollmentStatus.ACTIVE: + return _blocked( + BootstrapReasonCode.ENROLLMENT_NOT_ACTIVE, + f"enrollment is {enrollment.status.value}", + "status", + ) + + if revision is None or revision.revision_id != enrollment.study_revision_id: + return _blocked( + BootstrapReasonCode.REVISION_MISMATCH, + "enrollment is not bound to the requested revision", + "revision_id", + ) + if revision.status != RevisionStatus.PUBLISHED: + return _blocked( + BootstrapReasonCode.REVISION_NOT_PUBLISHED, + f"revision is {revision.status.value}", + "status", + ) + + protocol = StudyProtocolV1.model_validate(revision.protocol_json) + + window_issue = _schedule_issue(protocol, enrollment, timestamp) + if window_issue is not None: + return BootstrapResult( + outcome=BootstrapOutcome.BLOCKED, + reason=window_issue.code, + issue=window_issue, + ) + + if assignment is None or ( + assignment.enrollment_id != enrollment.enrollment_id + or assignment.study_revision_id != revision.revision_id + ): + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "assignment does not belong to this enrollment/revision", + "assignment_id", + ) + + condition = next( + ( + candidate + for candidate in protocol.conditions + if candidate.condition_id == assignment.condition_id + ), + None, + ) + if condition is None: + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + f"assigned condition {assignment.condition_id!r} is not in the revision", + "condition_id", + ) + + pin = condition.resolved_distribution + if pin is None: + return _blocked( + BootstrapReasonCode.RELEASE_NOT_FOUND, + "the assigned condition has no frozen distribution pin", + "resolved_distribution", + ) + + from research.study.agents.enums import DistributionMode + + is_byoa = pin.distribution_mode == DistributionMode.BYOA_EXTERNAL.value + + if release is None: + if not is_byoa: + return _blocked( + BootstrapReasonCode.RELEASE_NOT_FOUND, + "the pinned agent release is not registered", + "resolved_distribution", + ) + else: + if pin.agent_id and pin.agent_id != release.agent_id: + return _blocked( + BootstrapReasonCode.ARTIFACT_MISMATCH, + "release agent does not match the assigned condition pin", + "resolved_distribution.agent_id", + ) + if pin.release_id and pin.release_id != release.release_id: + return _blocked( + BootstrapReasonCode.ARTIFACT_MISMATCH, + "release id does not match the assigned condition pin", + "resolved_distribution.release_id", + ) + if pin.version and pin.version != release.version: + return _blocked( + BootstrapReasonCode.ARTIFACT_MISMATCH, + "release version does not match the assigned condition pin", + "resolved_distribution.version", + ) + + from research.study.agents.enums import QualificationStatus + + if not is_byoa: + # A PACKAGED release must be derived-qualified; a BYOA distribution is + # deliberately always unverified (it has no artifact to bind evidence to) + # and is launched from its frozen command/package identity instead. + if release is None or release.qualification_status != QualificationStatus.QUALIFIED: + qualification = ( + release.qualification_status.value + if release is not None + else "unregistered" + ) + return _blocked( + BootstrapReasonCode.RELEASE_NOT_QUALIFIED, + f"release is {qualification}", + "resolved_distribution", + ) + + selected_digest: Optional[str] = None + if is_byoa: + # BYOA: the participant installs the agent; there is no artifact to select + # or digest to pin. The manifest carries the command/package identity. + if not ( + pin.agent_package + or pin.agent_command + or (release is not None and release.byoa_identity) + ): + return _blocked( + BootstrapReasonCode.ARTIFACT_UNAVAILABLE, + "the BYOA distribution declares no command or agent package", + "resolved_distribution", + ) + elif release_resolver is not None: + resolution = release_resolver.resolve( + release.agent_id, release_id=release.release_id + ) + if resolution.status == ReleaseResolutionStatus.NOT_FOUND: + return _blocked( + BootstrapReasonCode.RELEASE_NOT_FOUND, + "the pinned release could not be resolved", + "resolved_distribution", + ) + if resolution.status != ReleaseResolutionStatus.RESOLVED: + return _blocked( + BootstrapReasonCode.RELEASE_NOT_QUALIFIED, + f"release resolution is {resolution.status.value}", + "resolved_distribution", + ) + selected_digest = resolution.artifact_digest + else: + if platform is not None: + artifact = release.artifact_for(platform[0], platform[1]) + else: + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + artifact = artifacts[0] if len(artifacts) == 1 else None + if artifact is None or not artifact.sha256: + return _blocked( + BootstrapReasonCode.ARTIFACT_UNAVAILABLE, + "no artifact is available for this platform", + "resolved_distribution", + ) + selected_digest = artifact.sha256 + + if not is_byoa and not selected_digest: + return _blocked( + BootstrapReasonCode.ARTIFACT_UNAVAILABLE, + "no resolvable artifact digest for the pinned release", + "resolved_distribution", + ) + + pinned_digest = pin.artifact_digest + if ( + not is_byoa + and pinned_digest + and pinned_digest != selected_digest + ): + return _blocked( + BootstrapReasonCode.ARTIFACT_MISMATCH, + "selected artifact does not match the frozen distribution pin", + "resolved_distribution.artifact_digest", + ) + + required_capabilities = protocol.environment_requirements.required_capabilities + if compatibility_result is None and required_capabilities: + return _blocked( + BootstrapReasonCode.COMPATIBILITY_MISSING, + "no compatibility result was supplied; failing closed", + "compatibility", + ) + if ( + compatibility_result is not None + and compatibility_result.decision != CompatibilityDecision.COMPATIBLE + ): + return _blocked( + BootstrapReasonCode.INCOMPATIBLE_ENVIRONMENT, + f"compatibility decision is {compatibility_result.decision.value}", + "compatibility", + ) + + telemetry = protocol.telemetry_policy + policies = BootstrapPolicies( + telemetry_policy=BootstrapTelemetryPolicy( + allowed_field_classes=[ + field_class.value for field_class in telemetry.allowed_field_classes + ], + content_capture=( + TelemetryFieldClass.CONTENT in telemetry.allowed_field_classes + ), + ), + privacy_policy=BootstrapPrivacyPolicy( + retention_action=protocol.privacy_policy.retention_action.value, + retention_days=protocol.privacy_policy.retention_days, + ), + session_policy=BootstrapSessionPolicy( + idle_timeout_seconds=protocol.session_policy.idle_timeout_seconds, + resume_grace_seconds=protocol.session_policy.resume_grace_seconds, + heartbeat_seconds=protocol.session_policy.heartbeat_seconds, + ), + ) + + research_session = session_factory.create_for_enrollment( + enrollment, revision, timestamp, context_id + ) + capability = issue_capability( + audience=signer.audience, + scope=signer.scope, + ttl_seconds=signer.capability_ttl_seconds, + revocation_epoch=enrollment.revocation_epoch, + secret=signer.secret, + now=timestamp, + enrollment_id=enrollment.enrollment_id, + research_session_id=research_session.research_session_id, + revision_id=revision.revision_id, + ) + + if release is not None and release.adapter is not None: + adapter_version = release.adapter.version + else: + adapter_version = condition.adapter_version + + if release is not None: + manifest_agent_id = release.agent_id + manifest_release_id = release.release_id + manifest_mode = release.distribution_mode.value + manifest_command = release.agent_command + manifest_command_args = list(release.agent_command_args) + manifest_package = release.agent_package + else: + # BYOA distribution with no registered release: launch from the frozen pin. + manifest_agent_id = ( + pin.agent_id or pin.agent_package or pin.agent_command or "" + ) + manifest_release_id = pin.release_id or "" + manifest_mode = pin.distribution_mode + manifest_command = pin.agent_command + manifest_command_args = list(pin.agent_command_args) + manifest_package = pin.agent_package + + draft = BootstrapManifestV1( + generated_at=timestamp, + study_id=revision.study_id, + revision_id=revision.revision_id, + revision_digest=revision.protocol_digest, + enrollment_id=enrollment.enrollment_id, + research_session=research_session, + assignment=BootstrapAssignment( + assignment_id=assignment.assignment_id, + condition_id=assignment.condition_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + protocol_digest=assignment.protocol_digest, + ), + agent_release=BootstrapAgentRelease( + agent_id=manifest_agent_id, + release_id=manifest_release_id, + artifact_digest=selected_digest or "", + adapter_version=adapter_version, + distribution_mode=manifest_mode, + agent_command=manifest_command, + agent_command_args=manifest_command_args, + agent_package=manifest_package, + ), + agent_profile=agent_profile, + policies=policies, + compatibility_receipt_ref=compatibility_ref, + compatibility=BootstrapCompatibility( + receipt_ref=compatibility_ref, + state=( + CoverageState.AVAILABLE + if compatibility_result is not None + else CoverageState.UNAVAILABLE + ), + reason=None if compatibility_result is not None else "NOT_REQUIRED", + ), + session_capability=capability, + ) + + signature = sign_manifest(draft, signer.secret) + signed = draft.model_copy( + update={ + "manifest_digest": signature.digest, + "signature": signature.signature, + } + ) + return BootstrapResult( + outcome=BootstrapOutcome.ISSUED, + reason=BootstrapReasonCode.OK, + manifest=signed, + ) diff --git a/src/research/runtime/bootstrap/signer.py b/src/research/runtime/bootstrap/signer.py new file mode 100644 index 0000000..419c01e --- /dev/null +++ b/src/research/runtime/bootstrap/signer.py @@ -0,0 +1,113 @@ +"""Dependency-free manifest signing and verification. + +The manifest digest is content-addressed over the canonical JSON of the manifest +with ``manifest_digest`` and the capability signature excluded. The signature is +an HMAC-SHA256 over that digest using an injected secret, so signing needs no +new dependency and no PKI. Server-side verification is authoritative. +""" + +from __future__ import annotations + +import hashlib +import hmac + +from research.canonical import canonical_bytes, canonical_hash + +from .models import ( + BootstrapManifestV1, + ManifestReasonCode, + ManifestSignature, + ManifestVerification, +) + +__all__ = [ + "compute_manifest_digest", + "manifest_digest_payload", + "sign_manifest", + "verify_manifest", +] + + +def manifest_digest_payload(manifest: BootstrapManifestV1) -> dict: + """Return the canonical mapping covered by the manifest digest. + + Excludes the ``manifest_digest`` and ``signature`` fields, so the digest and + its signature are stable and self-referential content is never hashed. + """ + data = manifest.model_dump(mode="json") + data.pop("manifest_digest", None) + data.pop("signature", None) + return data + + +def compute_manifest_digest(manifest: BootstrapManifestV1) -> str: + """Return the SHA-256 digest of the digest-covered canonical mapping.""" + return canonical_hash(manifest_digest_payload(manifest)) + + +def _hmac_hex(secret: str, message: str) -> str: + return hmac.new( + secret.encode("utf-8"), message.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + +def sign_manifest( + manifest: BootstrapManifestV1, secret: str +) -> ManifestSignature: + """Return ``(digest, signature)`` for a manifest.""" + digest = compute_manifest_digest(manifest) + return ManifestSignature(digest=digest, signature=_hmac_hex(secret, digest)) + + +def verify_manifest( + manifest: BootstrapManifestV1, secret: str +) -> ManifestVerification: + """Verify the manifest digest and signature, returning a typed reason.""" + expected = compute_manifest_digest(manifest) + if not manifest.manifest_digest: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_DIGEST_MISMATCH, + expected_digest=expected, + actual_digest="", + message="manifest_digest is missing", + ) + if manifest.manifest_digest != expected: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_DIGEST_MISMATCH, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest content does not match its digest", + ) + + signature = manifest.signature + if not signature: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MISSING_SIGNATURE, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest is not signed", + ) + + if not hmac.compare_digest(signature, _hmac_hex(secret, expected)): + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_SIGNATURE_MISMATCH, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest signature does not match", + ) + + return ManifestVerification( + ok=True, + reason=ManifestReasonCode.OK, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + ) + + +def canonical_manifest_bytes(manifest: BootstrapManifestV1) -> bytes: + """Convenience: canonical bytes of the manifest (signature excluded).""" + return canonical_bytes(manifest_digest_payload(manifest)) diff --git a/src/research/runtime/bootstrap/store.py b/src/research/runtime/bootstrap/store.py new file mode 100644 index 0000000..dd4da0e --- /dev/null +++ b/src/research/runtime/bootstrap/store.py @@ -0,0 +1,28 @@ +"""Persistence helpers for bootstrap session authorization. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App``. Session capabilities are signed and verified statelessly +(HMAC), so no capability row is written; the authoritative revocation epoch +lives on the enrollment row and is the only server-side state consulted here. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + +from database.research_schemas import ResearchEnrollment + + +def revocation_epoch_for( + session: Session, enrollment_id: uuid.UUID +) -> Optional[int]: + """Return the authoritative revocation epoch for an enrollment, or ``None``.""" + row = session.get(ResearchEnrollment, enrollment_id) + if row is None: + return None + return row.revocation_epoch diff --git a/src/research/runtime/sessions/__init__.py b/src/research/runtime/sessions/__init__.py new file mode 100644 index 0000000..4d086e4 --- /dev/null +++ b/src/research/runtime/sessions/__init__.py @@ -0,0 +1,78 @@ +"""Research session lifecycle and agent runs (Issue 07). + +Public surface: + +* :mod:`research.runtime.sessions.enums` - session states, close reasons, agent-run + outcomes and typed reason codes. +* :mod:`research.runtime.sessions.models` - ``ResearchSessionV1``, ``AgentRunV1``, + ``SessionPolicyV1``, transitions and typed results. +* :mod:`research.runtime.sessions.service` - the pure state machine (idle/resume values + are injected revision policy, never compiled constants). +* :mod:`research.runtime.sessions.store` - persistence adapters taking a caller-supplied + SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import AgentRunOutcome, CloseReason, SessionReasonCode, SessionState +from .models import ( + AgentRunResult, + AgentRunV1, + ResearchSessionV1, + ResumeResult, + SessionIssue, + SessionPolicyV1, + SessionResult, + SessionTransition, +) +from .service import ( + can_transition, + close, + end, + end_agent_run, + expire_if_idle, + go_offline, + on_agent_run_crashed, + on_qualifying_activity, + open_session, + policy_ref, + record_activity, + recover, + resume, + revoke, + session_policy_from_revision, + start_agent_run, + suspend, +) + +__all__ = [ + "AgentRunOutcome", + "AgentRunResult", + "AgentRunV1", + "CloseReason", + "ResearchSessionV1", + "ResumeResult", + "SessionIssue", + "SessionPolicyV1", + "SessionReasonCode", + "SessionResult", + "SessionState", + "SessionTransition", + "can_transition", + "close", + "end", + "end_agent_run", + "expire_if_idle", + "go_offline", + "on_agent_run_crashed", + "on_qualifying_activity", + "open_session", + "policy_ref", + "record_activity", + "recover", + "resume", + "revoke", + "session_policy_from_revision", + "start_agent_run", + "suspend", +] diff --git a/src/research/runtime/sessions/enums.py b/src/research/runtime/sessions/enums.py new file mode 100644 index 0000000..84e5b4e --- /dev/null +++ b/src/research/runtime/sessions/enums.py @@ -0,0 +1,74 @@ +"""Closed vocabularies for the research session lifecycle (Issue 07). + +Values match the Kotlin client contract in +``code4me2/src/main/kotlin/me/code4me/research/session`` so the server and the +plugin/proxy agree on the wire vocabulary. Members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "AgentRunOutcome", + "CloseReason", + "SessionReasonCode", + "SessionState", +] + + +class SessionState(str, Enum): + """Participant activity-session lifecycle states.""" + + NOT_STARTED = "not_started" + RUNNING = "running" + OFFLINE = "offline" + SUSPENDED = "suspended" + ENDED = "ended" + REVOKED = "revoked" + + @property + def is_terminal(self) -> bool: + """Whether this state is terminal (``ENDED`` or ``REVOKED``).""" + return self in (SessionState.ENDED, SessionState.REVOKED) + + +class CloseReason(str, Enum): + """Why a research session reached a terminal state.""" + + EXPLICIT_COMPLETION = "explicit_completion" + IDLE_TIMEOUT = "idle_timeout" + RESUME_GRACE_EXPIRED = "resume_grace_expired" + IDE_CLOSED = "ide_closed" + REVOKED = "revoked" + CRASH_RECOVERED = "crash_recovered" + MIGRATED = "migrated" + + +class AgentRunOutcome(str, Enum): + """Terminal outcome of one agent run.""" + + COMPLETED = "completed" + CRASHED = "crashed" + REVOKED = "revoked" + TERMINATED = "terminated" + UNKNOWN = "unknown" + + +class SessionReasonCode(str, Enum): + """Stable machine-readable reason for a session/run operation outcome.""" + + OK = "OK" + INVALID_STATE_TRANSITION = "INVALID_STATE_TRANSITION" + SESSION_TERMINAL = "SESSION_TERMINAL" + NOT_SUSPENDED = "NOT_SUSPENDED" + RESUME_GRACE_EXPIRED = "RESUME_GRACE_EXPIRED" + IDLE_TIMEOUT = "IDLE_TIMEOUT" + POLICY_MISSING = "POLICY_MISSING" + SESSION_NOT_RUNNING = "SESSION_NOT_RUNNING" + AGENT_RUN_TERMINAL = "AGENT_RUN_TERMINAL" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" diff --git a/src/research/runtime/sessions/models.py b/src/research/runtime/sessions/models.py new file mode 100644 index 0000000..6070ac2 --- /dev/null +++ b/src/research/runtime/sessions/models.py @@ -0,0 +1,173 @@ +"""Pydantic v2 contracts for the research session lifecycle (Issue 07). + +A ``ResearchSessionV1`` is the authoritative participant-activity boundary. An +``AgentRunV1`` is a distinct child identity: its id is never the session id, and +an agent restart inside the resume grace creates a new run without ending the +session. Idle/resume values live in :class:`SessionPolicyV1` (a revision policy +input), never as compiled constants. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, model_validator + +from .enums import ( + AgentRunOutcome, + CloseReason, + SessionReasonCode, + SessionState, +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class SessionPolicyV1(BaseModel): + """Revision-policy timing inputs for a session (never compiled constants).""" + + model_config = _BASE + + idle_timeout_seconds: int + resume_grace_seconds: int + heartbeat_seconds: int | None = None + + @model_validator(mode="after") + def _positive(self) -> SessionPolicyV1: + if self.idle_timeout_seconds <= 0: + raise ValueError("idle_timeout_seconds must be positive") + if self.resume_grace_seconds <= 0: + raise ValueError("resume_grace_seconds must be positive") + if self.heartbeat_seconds is not None and self.heartbeat_seconds <= 0: + raise ValueError("heartbeat_seconds must be positive when present") + return self + + +class ResearchSessionV1(BaseModel): + """One research session (authoritative participant-activity boundary).""" + + model_config = _FROZEN + + research_session_id: UUID + enrollment_id: UUID + study_revision_id: UUID + # Opaque execution-context id (project/window instance); never a path. + context_id: str = "" + state: SessionState = SessionState.NOT_STARTED + opened_at: datetime | None = None + last_activity_at: datetime | None = None + closed_at: datetime | None = None + close_reason: CloseReason | None = None + resume_generation: int = 0 + manifest_digest: str = "" + environment_ref: str | None = None + + @model_validator(mode="after") + def _consistency(self) -> ResearchSessionV1: + if self.resume_generation < 0: + raise ValueError("resume_generation must not be negative") + if self.state.is_terminal: + if self.closed_at is None or self.close_reason is None: + raise ValueError("a terminal session requires closed_at and close_reason") + elif self.closed_at is not None or self.close_reason is not None: + raise ValueError("a non-terminal session cannot have closed_at/close_reason") + return self + + +class AgentRunV1(BaseModel): + """One agent process inside a research session (distinct child identity).""" + + model_config = _FROZEN + + agent_run_id: UUID + research_session_id: UUID + agent_release_id: str | None = None + started_at: datetime + ended_at: datetime | None = None + outcome: AgentRunOutcome | None = None + + @model_validator(mode="after") + def _consistency(self) -> AgentRunV1: + if self.agent_run_id == self.research_session_id: + raise ValueError( + "AgentRun and ResearchSession identifiers must never be interchangeable" + ) + if self.ended_at is not None: + if self.ended_at < self.started_at: + raise ValueError("an agent run cannot end before it starts") + if self.outcome is None: + raise ValueError("a terminal agent run requires an outcome") + elif self.outcome is not None: + raise ValueError("a running agent run cannot have an outcome") + return self + + @property + def is_terminal(self) -> bool: + """Whether this run has ended.""" + return self.ended_at is not None + + +class SessionTransition(BaseModel): + """One recorded state transition (policy/evidence referenced).""" + + model_config = _FROZEN + + transition_id: UUID + research_session_id: UUID + from_state: SessionState + to_state: SessionState + occurred_at: datetime + reason: SessionReasonCode = SessionReasonCode.OK + close_reason: CloseReason | None = None + policy_ref: str | None = None + evidence_ref: str | None = None + + +class SessionIssue(BaseModel): + """One typed session/run rejection reason.""" + + model_config = _BASE + + code: SessionReasonCode + message: str + field: str = "" + + +class SessionResult(BaseModel): + """Typed outcome of a session operation (state + optional transition).""" + + model_config = _BASE + + accepted: bool + session: ResearchSessionV1 + transition: SessionTransition | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None + + +class ResumeResult(BaseModel): + """Typed outcome of a resume attempt.""" + + model_config = _BASE + + accepted: bool + reopened: bool + requires_new_session: bool + session: ResearchSessionV1 + transition: SessionTransition | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None + + +class AgentRunResult(BaseModel): + """Typed outcome of starting/ending an agent run.""" + + model_config = _BASE + + accepted: bool + run: AgentRunV1 | None = None + session: ResearchSessionV1 | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None diff --git a/src/research/runtime/sessions/service.py b/src/research/runtime/sessions/service.py new file mode 100644 index 0000000..0ff1801 --- /dev/null +++ b/src/research/runtime/sessions/service.py @@ -0,0 +1,559 @@ +"""Pure research-session state machine (Issue 07). + +Idle and resume values are injected revision-policy inputs +(:class:`~research.runtime.sessions.models.SessionPolicyV1`), never compiled constants. +Only the published lifecycle transitions are legal; every applied transition is +returned as a :class:`~research.runtime.sessions.models.SessionTransition` carrying the +reason and a policy/evidence reference. + +State diagram:: + + not_started -> running (first qualifying activity) + running -> offline (network unavailable) + offline -> running (heartbeat/upload recovers) + running -> suspended (IDE closes or sleeps) + suspended -> running (restart within resume grace; +1 generation) + suspended -> ended (resume grace expired) + running -> ended (explicit completion / idle timeout) + running -> revoked (withdrawal / consent pause) + offline -> revoked (server revocation observed) + ended, revoked -> (terminal) +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Callable, Optional + +from .enums import ( + AgentRunOutcome, + CloseReason, + SessionReasonCode, + SessionState, +) +from .models import ( + AgentRunResult, + AgentRunV1, + ResearchSessionV1, + ResumeResult, + SessionIssue, + SessionPolicyV1, + SessionResult, + SessionTransition, +) + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from research.study.protocol.models import StudyProtocolV1 + from research.study.protocol.publication import StudyRevision + +_LEGAL_TARGETS: dict[SessionState, frozenset[SessionState]] = { + SessionState.NOT_STARTED: frozenset({SessionState.RUNNING}), + SessionState.RUNNING: frozenset( + { + SessionState.OFFLINE, + SessionState.SUSPENDED, + SessionState.ENDED, + SessionState.REVOKED, + } + ), + SessionState.OFFLINE: frozenset( + {SessionState.RUNNING, SessionState.ENDED, SessionState.REVOKED} + ), + SessionState.SUSPENDED: frozenset({SessionState.RUNNING, SessionState.ENDED}), + SessionState.ENDED: frozenset(), + SessionState.REVOKED: frozenset(), +} + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: SessionReasonCode, message: str, field: str = "") -> SessionIssue: + return SessionIssue(code=code, message=message, field=field) + + +def _kill_switch_result( + session: ResearchSessionV1, kill_switch_check: Optional[Callable[[], bool]] +) -> Optional[SessionResult]: + """Return a non-accepted result when the injected kill switch is engaged. + + Returns ``None`` when no check is supplied or the switch is released, so the + caller proceeds with the pure state machine unchanged. + """ + if kill_switch_check is None or not kill_switch_check(): + return None + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.KILL_SWITCH_ENGAGED, + issue=_issue( + SessionReasonCode.KILL_SWITCH_ENGAGED, + "an operator kill switch is engaged for this scope", + "kill_switch", + ), + ) + + +def can_transition(from_state: SessionState, to_state: SessionState) -> bool: + """Whether ``to_state`` is a legal successor of ``from_state``.""" + return to_state in _LEGAL_TARGETS[from_state] + + +def policy_ref(policy: SessionPolicyV1) -> str: + """Human-readable, stable reference to the applied policy values.""" + return ( + f"idle_timeout_seconds={policy.idle_timeout_seconds}," + f"resume_grace_seconds={policy.resume_grace_seconds}" + ) + + +def session_policy_from_revision( + protocol: StudyProtocolV1, +) -> Optional[SessionPolicyV1]: + """Extract a usable :class:`SessionPolicyV1` from a revision, or ``None``. + + ``None`` means the revision does not declare both the idle timeout and the + resume grace, so no timing decision can be made. The server never falls back + to a compiled default. + """ + session_policy = protocol.session_policy + if ( + session_policy.idle_timeout_seconds is None + or session_policy.resume_grace_seconds is None + ): + return None + return SessionPolicyV1( + idle_timeout_seconds=session_policy.idle_timeout_seconds, + resume_grace_seconds=session_policy.resume_grace_seconds, + heartbeat_seconds=session_policy.heartbeat_seconds, + ) + + +def open_session( + enrollment: Enrollment, + revision: StudyRevision, + *, + manifest_digest: str, + environment_ref: Optional[str] = None, + context_id: str = "", + now: Optional[datetime] = None, +) -> ResearchSessionV1: + """Create a new, not-yet-started session bound to an enrollment/revision.""" + return ResearchSessionV1( + research_session_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_revision_id=revision.revision_id, + context_id=context_id, + state=SessionState.NOT_STARTED, + opened_at=None, + last_activity_at=None, + closed_at=None, + close_reason=None, + resume_generation=0, + manifest_digest=manifest_digest, + environment_ref=environment_ref, + ) + + +def _apply( + session: ResearchSessionV1, + to_state: SessionState, + now: datetime, + *, + reason: SessionReasonCode = SessionReasonCode.OK, + close_reason: Optional[CloseReason] = None, + policy_reference: Optional[str] = None, + evidence_ref: Optional[str] = None, + resume_generation: Optional[int] = None, +) -> SessionResult: + if not can_transition(session.state, to_state): + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.INVALID_STATE_TRANSITION, + issue=_issue( + SessionReasonCode.INVALID_STATE_TRANSITION, + f"illegal session transition {session.state.value} -> {to_state.value}", + "state", + ), + ) + + updates: dict = {"state": to_state, "last_activity_at": now} + if to_state == SessionState.RUNNING: + updates["opened_at"] = session.opened_at or now + if to_state.is_terminal: + updates["closed_at"] = now + updates["close_reason"] = close_reason or ( + CloseReason.REVOKED + if to_state == SessionState.REVOKED + else CloseReason.EXPLICIT_COMPLETION + ) + if resume_generation is not None: + updates["resume_generation"] = resume_generation + + updated = session.model_copy(update=updates) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=session.research_session_id, + from_state=session.state, + to_state=to_state, + occurred_at=now, + reason=reason, + close_reason=updated.close_reason, + policy_ref=policy_reference, + evidence_ref=evidence_ref, + ) + return SessionResult( + accepted=True, session=updated, transition=transition, reason=reason + ) + + +def record_activity( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """Update the last-activity marker without changing state.""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + return SessionResult( + accepted=True, + session=session.model_copy(update={"last_activity_at": timestamp}), + reason=SessionReasonCode.OK, + ) + + +def on_qualifying_activity( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> SessionResult: + """First qualifying activity: ``not_started -> running``.""" + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + "a terminal session cannot accept new activity", + "state", + ), + ) + if session.state == SessionState.NOT_STARTED: + return _apply(session, SessionState.RUNNING, timestamp) + return record_activity(session, timestamp) + + +def go_offline( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """``running -> offline`` (network unavailable).""" + return _apply(session, SessionState.OFFLINE, _now(now)) + + +def recover( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """``offline -> running`` (heartbeat/upload recovers).""" + return _apply(session, SessionState.RUNNING, _now(now)) + + +def suspend( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """``running -> suspended`` (IDE closes or the machine sleeps).""" + return _apply( + session, + SessionState.SUSPENDED, + _now(now), + evidence_ref=evidence_ref, + ) + + +def end( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + close_reason: CloseReason = CloseReason.EXPLICIT_COMPLETION, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """End a live session (explicit completion or a caller-chosen reason).""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + return _apply( + session, + SessionState.ENDED, + timestamp, + close_reason=close_reason, + evidence_ref=evidence_ref, + ) + + +def revoke( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """Withdrawal/consent pause: terminal revocation.""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + return _apply( + session, + SessionState.REVOKED, + timestamp, + close_reason=CloseReason.REVOKED, + evidence_ref=evidence_ref, + ) + + +def resume( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + policy: SessionPolicyV1, +) -> ResumeResult: + """Reopen a suspended session within its resume grace, or end it once.""" + timestamp = _now(now) + + if session.state.is_terminal: + return ResumeResult( + accepted=False, + reopened=False, + requires_new_session=True, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + if session.state != SessionState.SUSPENDED: + return ResumeResult( + accepted=False, + reopened=False, + requires_new_session=False, + session=session, + reason=SessionReasonCode.NOT_SUSPENDED, + issue=_issue( + SessionReasonCode.NOT_SUSPENDED, + f"session is {session.state.value}, not suspended", + "state", + ), + ) + + reference = policy_ref(policy) + last_activity = session.last_activity_at + within_grace = ( + last_activity is not None + and (timestamp - last_activity).total_seconds() <= policy.resume_grace_seconds + ) + + if within_grace: + result = _apply( + session, + SessionState.RUNNING, + timestamp, + policy_reference=reference, + resume_generation=session.resume_generation + 1, + ) + return ResumeResult( + accepted=result.accepted, + reopened=result.accepted, + requires_new_session=False, + session=result.session, + transition=result.transition, + reason=result.reason, + issue=result.issue, + ) + + ended = session.model_copy( + update={ + "state": SessionState.ENDED, + "closed_at": timestamp, + "close_reason": CloseReason.RESUME_GRACE_EXPIRED, + "last_activity_at": timestamp, + } + ) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=session.research_session_id, + from_state=session.state, + to_state=SessionState.ENDED, + occurred_at=timestamp, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + close_reason=CloseReason.RESUME_GRACE_EXPIRED, + policy_ref=reference, + ) + return ResumeResult( + accepted=True, + reopened=False, + requires_new_session=True, + session=ended, + transition=transition, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + ) + + +def expire_if_idle( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + policy: SessionPolicyV1, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> SessionResult: + """End a live session when it exceeds the revision idle timeout.""" + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if session.state.is_terminal or session.state == SessionState.NOT_STARTED: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + last_activity = session.last_activity_at + if last_activity is None: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + if (timestamp - last_activity).total_seconds() <= policy.idle_timeout_seconds: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.IDLE_TIMEOUT, + close_reason=CloseReason.IDLE_TIMEOUT, + policy_reference=policy_ref(policy), + ) + + +def close( + session: ResearchSessionV1, + reason: CloseReason, + now: Optional[datetime] = None, + *, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> SessionResult: + """Apply a typed close reason to a session.""" + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if reason == CloseReason.REVOKED: + return revoke(session, timestamp, evidence_ref=f"close_reason={reason.value}") + if reason == CloseReason.IDLE_TIMEOUT: + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.IDLE_TIMEOUT, + close_reason=CloseReason.IDLE_TIMEOUT, + evidence_ref=f"close_reason={reason.value}", + ) + if reason == CloseReason.RESUME_GRACE_EXPIRED: + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + close_reason=CloseReason.RESUME_GRACE_EXPIRED, + evidence_ref=f"close_reason={reason.value}", + ) + return end( + session, + timestamp, + close_reason=reason, + evidence_ref=f"close_reason={reason.value}", + ) + + +def start_agent_run( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + agent_release_id: Optional[str] = None, +) -> AgentRunResult: + """Start an agent run; allowed only while the session is ``RUNNING``.""" + timestamp = _now(now) + if session.state != SessionState.RUNNING: + return AgentRunResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_NOT_RUNNING, + issue=_issue( + SessionReasonCode.SESSION_NOT_RUNNING, + f"an agent run requires a RUNNING session, not {session.state.value}", + "state", + ), + ) + run = AgentRunV1( + agent_run_id=uuid.uuid4(), + research_session_id=session.research_session_id, + agent_release_id=agent_release_id, + started_at=timestamp, + ) + return AgentRunResult( + accepted=True, run=run, session=session, reason=SessionReasonCode.OK + ) + + +def end_agent_run( + run: AgentRunV1, + outcome: AgentRunOutcome, + now: Optional[datetime] = None, +) -> AgentRunResult: + """End an agent run; idempotent once terminal.""" + timestamp = _now(now) + if run.is_terminal: + return AgentRunResult( + accepted=True, run=run, session=None, reason=SessionReasonCode.AGENT_RUN_TERMINAL + ) + ended = run.model_copy(update={"ended_at": timestamp, "outcome": outcome}) + return AgentRunResult( + accepted=True, run=ended, session=None, reason=SessionReasonCode.OK + ) + + +def on_agent_run_crashed( + run: AgentRunV1, + session: ResearchSessionV1, + now: Optional[datetime] = None, +) -> AgentRunResult: + """Crash handling: end the run; the research session is preserved.""" + result = end_agent_run(run, AgentRunOutcome.CRASHED, now) + return AgentRunResult( + accepted=result.accepted, + run=result.run, + session=session, + reason=result.reason, + issue=result.issue, + ) diff --git a/src/research/runtime/sessions/store.py b/src/research/runtime/sessions/store.py new file mode 100644 index 0000000..4471af9 --- /dev/null +++ b/src/research/runtime/sessions/store.py @@ -0,0 +1,316 @@ +"""CRUD-style persistence helpers for research sessions and agent runs. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. Session and agent-run +rows are distinct tables/identities; the store never treats one id as the other. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import ( + AgentRunV1, + ResearchSessionV1, + SessionTransition, + ) + +from database.research_schemas import ResearchAgentRun +from database.research_schemas import ResearchSessionV1 as ResearchSessionRow + +from .enums import AgentRunOutcome, CloseReason, SessionState + +_TERMINAL_STATES = (SessionState.ENDED.value, SessionState.REVOKED.value) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _session_row(research_session: ResearchSessionV1) -> ResearchSessionRow: + return ResearchSessionRow( + session_id=research_session.research_session_id, + enrollment_id=research_session.enrollment_id, + study_revision_id=research_session.study_revision_id, + context_id=research_session.context_id or str(research_session.research_session_id), + state=research_session.state.value, + opened_at=research_session.opened_at, + last_activity_at=research_session.last_activity_at, + closed_at=research_session.closed_at, + close_reason=( + research_session.close_reason.value + if research_session.close_reason is not None + else None + ), + resume_generation=research_session.resume_generation, + manifest_digest=research_session.manifest_digest, + environment_json={ + "environment_ref": research_session.environment_ref + }, + created_at=_now(), + ) + + +def create_session( + session: Session, + research_session: ResearchSessionV1, + *, + commit: bool = True, +) -> ResearchSessionRow: + """Insert a new research session row, atomically. + + The partial unique index ``uq_research_session_active_context`` allows at + most one non-terminal session per ``(enrollment_id, context_id)``. A savepoint + around the insert means a concurrent create that loses the race rolls back + only its own insert and returns the winning session for that same context + instead of leaving a duplicate (or aborting the caller's transaction). + + ``commit=False`` lets bootstrap own one unit of work (assignment + session + commit together, or neither). + """ + row = _session_row(research_session) + try: + with session.begin_nested(): + session.add(row) + except IntegrityError: + existing = get_active_session_for_context( + session, research_session.enrollment_id, row.context_id + ) + if existing is not None: + return existing + raise + if commit: + session.commit() + else: + session.flush() + session.refresh(row) + return row + + +def get_session( + session: Session, research_session_id: uuid.UUID, *, for_update: bool = False +) -> Optional[ResearchSessionRow]: + """Fetch a session row by id, or ``None``. + + ``for_update`` takes a row lock (``SELECT ... FOR UPDATE``) so ingestion can + serialize concurrent batches for the same session. + """ + if not for_update: + return session.get(ResearchSessionRow, research_session_id) + statement = ( + select(ResearchSessionRow) + .where(ResearchSessionRow.session_id == research_session_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def list_sessions( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[ResearchSessionRow]: + """List sessions, optionally for one enrollment, newest-first.""" + statement = select(ResearchSessionRow) + if enrollment_id is not None: + statement = statement.where(ResearchSessionRow.enrollment_id == enrollment_id) + statement = statement.order_by(ResearchSessionRow.created_at.desc()) + return list(session.execute(statement).scalars().all()) + + +def get_active_session_for_enrollment( + session: Session, enrollment_id: uuid.UUID +) -> Optional[ResearchSessionRow]: + """Return the newest non-terminal session for an enrollment, or ``None``.""" + statement = ( + select(ResearchSessionRow) + .where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.state.notin_(_TERMINAL_STATES), + ) + .order_by(ResearchSessionRow.created_at.desc()) + ) + return session.execute(statement).scalars().first() + + +def get_active_session_for_context( + session: Session, enrollment_id: uuid.UUID, context_id: str +) -> Optional[ResearchSessionRow]: + """Return the non-terminal session for ``(enrollment, context)``, or ``None``. + + This is the idempotent-creation lookup: one authenticated execution context + maps to at most one live session. + """ + statement = ( + select(ResearchSessionRow) + .where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.context_id == context_id, + ResearchSessionRow.state.notin_(_TERMINAL_STATES), + ) + .order_by(ResearchSessionRow.created_at.desc()) + ) + return session.execute(statement).scalars().first() + + +def update_session( + session: Session, research_session: ResearchSessionV1 +) -> Optional[ResearchSessionRow]: + """Persist a state/timestamp/close change onto an existing session row.""" + row = session.get(ResearchSessionRow, research_session.research_session_id) + if row is None: + return None + row.state = research_session.state.value + row.opened_at = research_session.opened_at + row.last_activity_at = research_session.last_activity_at + row.closed_at = research_session.closed_at + row.close_reason = ( + research_session.close_reason.value + if research_session.close_reason is not None + else None + ) + row.resume_generation = research_session.resume_generation + row.manifest_digest = research_session.manifest_digest + row.environment_json = {"environment_ref": research_session.environment_ref} + session.commit() + session.refresh(row) + return row + + +def insert_transition( + session: Session, transition: SessionTransition +) -> SessionTransition: + """Append one recorded state transition to its session's log. + + The transition log is ``research_session.transitions_json`` (the session is + the only owner of its transitions), so no child row is written. + """ + row = session.get(ResearchSessionRow, transition.research_session_id) + if row is None: # pragma: no cover - guarded by the session router + raise ValueError(f"research session {transition.research_session_id} not found") + existing = row.transitions_json + serialized = list(existing) if isinstance(existing, list) else [] + serialized.append(transition.model_dump(mode="json")) + row.transitions_json = serialized + session.add(row) + session.commit() + session.refresh(row) + return transition + + +def create_agent_run(session: Session, run: AgentRunV1) -> ResearchAgentRun: + """Insert a new agent-run row (distinct id from its session).""" + row = ResearchAgentRun( + agent_run_id=run.agent_run_id, + research_session_id=run.research_session_id, + agent_release_id=run.agent_release_id, + started_at=run.started_at, + ended_at=run.ended_at, + outcome=run.outcome.value if run.outcome is not None else None, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_agent_run( + session: Session, agent_run_id: uuid.UUID +) -> Optional[ResearchAgentRun]: + """Fetch an agent-run row by id, or ``None``.""" + return session.get(ResearchAgentRun, agent_run_id) + + +def list_agent_runs( + session: Session, research_session_id: uuid.UUID +) -> Sequence[ResearchAgentRun]: + """List the agent runs of one session, oldest-first.""" + statement = ( + select(ResearchAgentRun) + .where(ResearchAgentRun.research_session_id == research_session_id) + .order_by(ResearchAgentRun.started_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_session(row: ResearchSessionRow) -> ResearchSessionV1: + """Rehydrate a session row into the domain model.""" + from .models import ResearchSessionV1 as ResearchSessionModel + + state = row.state + close_reason = row.close_reason + environment = row.environment_json or {} + return ResearchSessionModel( + research_session_id=row.session_id, + enrollment_id=row.enrollment_id, + study_revision_id=row.study_revision_id, + context_id=row.context_id or "", + state=state if isinstance(state, SessionState) else SessionState(state), + opened_at=row.opened_at, + last_activity_at=row.last_activity_at, + closed_at=row.closed_at, + close_reason=( + close_reason + if isinstance(close_reason, CloseReason) or close_reason is None + else CloseReason(close_reason) + ), + resume_generation=row.resume_generation, + manifest_digest=row.manifest_digest, + environment_ref=environment.get("environment_ref"), + ) + + +def row_to_agent_run(row: ResearchAgentRun) -> AgentRunV1: + """Rehydrate an agent-run row into the domain model.""" + from .models import AgentRunV1 as AgentRunModel + + outcome = row.outcome + return AgentRunModel( + agent_run_id=row.agent_run_id, + research_session_id=row.research_session_id, + agent_release_id=row.agent_release_id, + started_at=row.started_at, + ended_at=row.ended_at, + outcome=( + outcome + if isinstance(outcome, AgentRunOutcome) or outcome is None + else AgentRunOutcome(outcome) + ), + ) + + +def row_to_transition(row: Mapping[str, Any]) -> SessionTransition: + """Rehydrate one serialized ``transitions_json`` entry.""" + from .models import SessionTransition as SessionTransitionModel + + return SessionTransitionModel.model_validate(dict(row)) + + +def session_summary(row: ResearchSessionRow) -> dict[str, Any]: + """Return a compact, non-secret session summary safe for responses.""" + environment = row.environment_json or {} + return { + "research_session_id": str(row.session_id), + "enrollment_id": str(row.enrollment_id), + "study_revision_id": str(row.study_revision_id), + "context_id": row.context_id, + "state": row.state, + "opened_at": _iso(row.opened_at), + "last_activity_at": _iso(row.last_activity_at), + "closed_at": _iso(row.closed_at), + "close_reason": row.close_reason, + "resume_generation": row.resume_generation, + "manifest_digest": row.manifest_digest, + "environment_ref": environment.get("environment_ref"), + } + + +def _iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None diff --git a/src/research/study/__init__.py b/src/research/study/__init__.py new file mode 100644 index 0000000..d6e7c5d --- /dev/null +++ b/src/research/study/__init__.py @@ -0,0 +1,13 @@ +"""Study definition, agent registry and runtime packaging. + +Merged home for the versioned study protocol/publication service, the agent +release registry and capability snapshots, and the runtime packaging and +conformance suite. The individual sub-packages keep their original public +surfaces: + +* :mod:`research.study.protocol` +* :mod:`research.study.agents` +* :mod:`research.study.packaging` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/study/agents/__init__.py b/src/research/study/agents/__init__.py new file mode 100644 index 0000000..570ce5c --- /dev/null +++ b/src/research/study/agents/__init__.py @@ -0,0 +1,84 @@ +"""Agent registry and capability contract (Issue 04). + +Public surface: + +* :mod:`research.study.agents.enums` - closed vocabularies for qualification status, + distribution source, snapshot capability state, coverage state, registry + reason codes, plus the re-exported Issue 01 ``Fidelity``. +* :mod:`research.study.agents.models` - Pydantic v2 contracts for + :class:`~research.study.agents.models.AgentReleaseV1` (distribution + adapter + identity) and :class:`~research.study.agents.models.CapabilitySnapshotV1` (separate + declared/observed maps with explicit ``UNKNOWN``/``UNAVAILABLE``). +* :mod:`research.study.agents.registry` - the pure :class:`~research.study.agents.registry.AgentRegistry` + service: digest-identity registration, qualification transitions, exact + platform resolution, qualification assessment and capability coverage. +* :mod:`research.study.agents.resolver` - :class:`~research.study.agents.resolver.RegistryReleaseResolver`, + the bridge implementing :class:`research.study.protocol.validation.ReleaseResolver` + so protocol publication rejects missing/unqualified/withdrawn releases. + +The persistence adapters live in :mod:`research.study.agents.store` and take a +caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" + +from .enums import ( + CapabilityCoverageState, + DistributionSourceType, + Fidelity, + QualificationStatus, + RegistryReasonCode, + SnapshotCapabilityState, +) +from .models import ( + AdapterRef, + AgentReleaseV1, + CapabilityCoverage, + CapabilityCoverageReport, + CapabilityEntry, + CapabilitySnapshotV1, + DistributionArtifact, + EnvironmentRef, + ReleaseDisplay, + SnapshotFailure, +) +from .registry import ( + AgentRegistry, + ArtifactResolution, + QualificationAssessment, + RegistrationResult, + RegistryIssue, + build_capability_snapshot, + capability_coverage, + coverage_report, + derive_qualification_status, +) +from .resolver import RegistryReleaseResolver + +__all__ = [ + "AdapterRef", + "AgentRegistry", + "AgentReleaseV1", + "ArtifactResolution", + "CapabilityCoverage", + "CapabilityCoverageReport", + "CapabilityCoverageState", + "CapabilityEntry", + "CapabilitySnapshotV1", + "DistributionArtifact", + "DistributionSourceType", + "EnvironmentRef", + "Fidelity", + "QualificationAssessment", + "QualificationStatus", + "RegistrationResult", + "RegistryIssue", + "RegistryReasonCode", + "RegistryReleaseResolver", + "ReleaseDisplay", + "SnapshotCapabilityState", + "SnapshotFailure", + "build_capability_snapshot", + "capability_coverage", + "coverage_report", + "derive_qualification_status", +] diff --git a/src/research/study/agents/distributions.py b/src/research/study/agents/distributions.py new file mode 100644 index 0000000..5a95123 --- /dev/null +++ b/src/research/study/agents/distributions.py @@ -0,0 +1,224 @@ +"""Resolve an ``AgentProfile`` *distribution* into a non-secret view. + +An :class:`~database.db_schemas.AgentProfile` IS a distribution: it names the +exact agent artifact a study condition runs. This module turns a profile row plus +its (optional) registry release into the +:class:`~research.study.protocol.validation.DistributionResolution` the protocol +validator consumes, and into the derived, read-only shape the admin UI needs. + +The resolution is deliberately database-agnostic: callers pass an already-loaded +profile and release, so this stays usable from the backend router, the seed +tooling and tests without importing ``App``. + +Verification is **derived**, never stored: + +* ``PACKAGED`` is verified only when its release carries a passing conformance + receipt (``derive_qualification_status(release) == QUALIFIED``). +* ``BYOA_EXTERNAL`` is *always* unverified: a participant-installed agent has no + artifact to bind conformance evidence to. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Optional + +from research.study.protocol.enums import ReleaseResolutionStatus +from research.study.protocol.models import ResolvedAgentConfig +from research.study.protocol.validation import DistributionResolution + +from .enums import DistributionMode, QualificationStatus +from .registry import AgentRegistry +from .resolver import RegistryReleaseResolver + +if TYPE_CHECKING: + from .models import AgentReleaseV1 + +__all__ = [ + "distribution_supported_platforms", + "parse_command_args", + "resolve_distribution_view", +] + +_WITHDRAWN = frozenset( + {QualificationStatus.RETIRED, QualificationStatus.BLOCKED} +) + + +def parse_command_args(value: Any) -> list[str]: + """Normalize a stored ``agent_command_args`` value into a string list. + + The column is JSON text, but callers may hand a list (a freshly built + payload) or ``None``. Anything unparseable degrades to an empty list rather + than raising, because it is only ever an argument vector. + """ + if value is None: + return [] + if isinstance(value, (list, tuple)): + return [str(item) for item in value] + if isinstance(value, str): + try: + parsed = json.loads(value) + except (TypeError, ValueError): + return [] + if isinstance(parsed, list): + return [str(item) for item in parsed] + return [] + + +def _packaged_digest( + release: AgentReleaseV1, platform: Optional[tuple[str, str]] +) -> Optional[str]: + registry = AgentRegistry() + if not registry.register_release(release).accepted: + return None + resolver = RegistryReleaseResolver(registry, platform=platform) + return resolver.artifact_digest(release) + + +def _agent_config( + profile: Any, + release: Optional[AgentReleaseV1], + connection: Any, + funding_owner_user_id: Any = None, +) -> Optional[ResolvedAgentConfig]: + """Freeze the profile's non-secret executable config, when available. + + A lightweight stand-in (no ``model``/``tools_json``) has no executable config + to freeze, so this returns ``None``; such a revision is refused at execution + time rather than read from a mutable profile row. + """ + if getattr(profile, "model", None) is None: + return None + profile_id = getattr(profile, "profile_id", None) or getattr( + profile, "distribution_id", None + ) + if profile_id is None: + return None + return ResolvedAgentConfig( + profile_id=profile_id, + name=str(getattr(profile, "name", "") or ""), + model=str(getattr(profile, "model", "") or ""), + framework_version=str(getattr(profile, "framework_version", "") or ""), + tools_json=str(getattr(profile, "tools_json", "") or "[]"), + approval_policy=str(getattr(profile, "approval_policy", "") or ""), + max_steps=int(getattr(profile, "max_steps", 0) or 0), + temperature=getattr(profile, "temperature", None), + max_context_tokens=getattr(profile, "max_context_tokens", None), + connection_id=getattr(profile, "connection_id", None), + connection_label=getattr(connection, "label", None), + funding_owner_user_id=funding_owner_user_id, + ) + + +def resolve_distribution_view( + profile: Any, + release: Optional[AgentReleaseV1], + *, + distribution_id: Any = None, + platform: Optional[tuple[str, str]] = None, + connection: Any = None, + funding_owner_user_id: Any = None, +) -> DistributionResolution: + """Build the validator's view of one distribution. + + ``profile`` is any object exposing ``distribution_mode``, ``release_id``, + ``agent_package``, ``agent_command`` and ``agent_command_args`` (an + ``AgentProfile`` row or a lightweight stand-in). ``release`` is the already + rehydrated registry release for ``release_id``, or ``None`` when it does not + exist. + + When ``profile`` also exposes the executable config (a real profile row), + that non-secret config is attached as ``agent_config`` so publication can + freeze it into the immutable revision. ``funding_owner_user_id`` is the study + owner whose connection grant funds execution. + """ + if distribution_id is None: + distribution_id = getattr(profile, "profile_id", None) + raw_mode = str(getattr(profile, "distribution_mode", "") or "PACKAGED") + mode = raw_mode.strip().upper() + release_id = getattr(profile, "release_id", None) + agent_package = getattr(profile, "agent_package", None) + agent_command = getattr(profile, "agent_command", None) + agent_command_args = parse_command_args( + getattr(profile, "agent_command_args", None) + ) + agent_config = _agent_config( + profile, release, connection, funding_owner_user_id + ) + + if mode == DistributionMode.BYOA_EXTERNAL.value: + # A BYOA distribution is identity-pinned and can never be verified. + identity_present = bool( + agent_package or agent_command or (release is not None) + ) + return DistributionResolution( + found=True, + distribution_id=distribution_id, + distribution_mode=mode, + release_id=release_id, + agent_id=release.agent_id if release is not None else None, + version=release.version if release is not None else None, + agent_package=agent_package + or (release.agent_package if release is not None else None), + agent_command=agent_command + or (release.agent_command if release is not None else None), + agent_command_args=agent_command_args + or (list(release.agent_command_args) if release is not None else []), + artifact_digest=None, + verified=False, + release_status=( + ReleaseResolutionStatus.RESOLVED + if identity_present + else ReleaseResolutionStatus.UNQUALIFIED + ), + message=( + None + if identity_present + else "BYOA distribution declares neither a package nor a command" + ), + agent_config=agent_config, + ) + + view = DistributionResolution( + found=True, + distribution_id=distribution_id, + distribution_mode=mode or DistributionMode.PACKAGED.value, + release_id=release_id, + agent_package=None, + agent_command=None, + artifact_digest=None, + verified=False, + agent_config=agent_config, + ) + if not release_id: + return view + if release is None: + view.message = "release is not registered" + return view + + view.agent_id = release.agent_id + view.version = release.version + if release.qualification_status in _WITHDRAWN: + view.release_status = ReleaseResolutionStatus.WITHDRAWN + view.message = f"release is {release.qualification_status.value}" + return view + if release.qualification_status != QualificationStatus.QUALIFIED: + view.release_status = ReleaseResolutionStatus.UNQUALIFIED + view.message = "release is not qualified" + return view + + view.verified = True + view.release_status = ReleaseResolutionStatus.RESOLVED + view.artifact_digest = _packaged_digest(release, platform) + return view + + +def distribution_supported_platforms( + release: Optional[AgentReleaseV1], +) -> list[dict[str, str]]: + """The ``(os, arch)`` pairs a release publishes, in a stable order.""" + if release is None: + return [] + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + return [{"os": artifact.os, "arch": artifact.arch} for artifact in artifacts] diff --git a/src/research/study/agents/enums.py b/src/research/study/agents/enums.py new file mode 100644 index 0000000..041bbfb --- /dev/null +++ b/src/research/study/agents/enums.py @@ -0,0 +1,116 @@ +"""Closed vocabularies for the agent registry and capability contract (Issue 04). + +Every member is part of a persisted contract (release JSON, snapshot JSON, +registry responses), so members are additive only: renaming or removing a +member is a breaking change to already-stored releases and snapshots. + +``Fidelity`` is re-exported from :mod:`research.compatibility.enums` so the +registry shares the exact Issue 01 fidelity vocabulary instead of inventing a +parallel one. +""" + +from __future__ import annotations + +from enum import Enum + +from research.compatibility.enums import Fidelity # noqa: F401 - intentional re-export + +__all__ = [ + "CapabilityCoverageState", + "DistributionMode", + "DistributionSourceType", + "Fidelity", + "QualificationStatus", + "RegistryReasonCode", + "SnapshotCapabilityState", +] + + +class DistributionMode(str, Enum): + """How a release's agent reaches the participant host. + + ``PACKAGED`` (the default) is the historical, digest-pinned contract: the + agent is a packaged artifact shipped inside the plugin runtime and the + release carries a verified ``sha256`` per platform. ``BYOA_EXTERNAL`` means + "bring your own agent": the participant installs the agent (Goose, Codex, ...) + and the release pins a command/package identity instead of an artifact digest. + The two modes are persisted on the release and embedded in every bootstrap + manifest so the plugin never silently mixes them. + """ + + PACKAGED = "PACKAGED" + BYOA_EXTERNAL = "BYOA_EXTERNAL" + + +class QualificationStatus(str, Enum): + """Review status of one agent release. + + Distribution, protocol capability and observability capability are + separate contracts. ``QUALIFIED`` is **derived** from verified conformance + evidence (a passing receipt bound to the release's artifact/adapter); a + release without such evidence is ``UNQUALIFIED``. ``DRAFT``, + ``CONDITIONALLY_QUALIFIED``, ``RETIRED`` and ``BLOCKED`` remain in the + persisted vocabulary for historical rows and for the withdrawal mapping. + """ + + UNQUALIFIED = "UNQUALIFIED" + DRAFT = "DRAFT" + QUALIFIED = "QUALIFIED" + CONDITIONALLY_QUALIFIED = "CONDITIONALLY_QUALIFIED" + RETIRED = "RETIRED" + BLOCKED = "BLOCKED" + + +class DistributionSourceType(str, Enum): + """Where a release's distribution metadata came from.""" + + EXTERNAL_REGISTRY = "EXTERNAL_REGISTRY" + RESEARCH_OVERLAY = "RESEARCH_OVERLAY" + BUNDLED = "BUNDLED" + + +class SnapshotCapabilityState(str, Enum): + """Capability state captured in a runtime snapshot. + + ``UNKNOWN``, ``DECLARED``, ``UNAVAILABLE``, ``PARTIAL`` and ``BROKEN`` are + deliberately distinct from ``OBSERVED``: declared intent and observed + behavior are separate facts, and an absent observation is never promoted to + a supported one. + """ + + UNKNOWN = "UNKNOWN" + DECLARED = "DECLARED" + OBSERVED = "OBSERVED" + UNAVAILABLE = "UNAVAILABLE" + PARTIAL = "PARTIAL" + BROKEN = "BROKEN" + + +class CapabilityCoverageState(str, Enum): + """Aggregate observability of one capability across declared vs observed.""" + + OBSERVED = "OBSERVED" + DECLARED_ONLY = "DECLARED_ONLY" + UNKNOWN = "UNKNOWN" + UNAVAILABLE = "UNAVAILABLE" + PARTIAL = "PARTIAL" + BROKEN = "BROKEN" + + +class RegistryReasonCode(str, Enum): + """Stable machine-readable reason a registry operation was rejected.""" + + DIGEST_MISMATCH = "DIGEST_MISMATCH" + UNSUPPORTED_PLATFORM = "UNSUPPORTED_PLATFORM" + NOT_QUALIFIED = "NOT_QUALIFIED" + WITHDRAWN = "WITHDRAWN" + DUPLICATE_RELEASE = "DUPLICATE_RELEASE" + INVALID_TRANSITION = "INVALID_TRANSITION" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" + ARTIFACT_MISSING = "ARTIFACT_MISSING" + INCOMPLETE_ARTIFACTS = "INCOMPLETE_ARTIFACTS" + ADAPTER_INCOMPATIBLE = "ADAPTER_INCOMPATIBLE" + INVALID_VERSION_RANGE = "INVALID_VERSION_RANGE" + SNAPSHOT_INVALID = "SNAPSHOT_INVALID" + # A BYOA release declares neither a command nor a package identity to resolve. + AGENT_NOT_FOUND = "AGENT_NOT_FOUND" diff --git a/src/research/study/agents/manifest_import.py b/src/research/study/agents/manifest_import.py new file mode 100644 index 0000000..8a8bcb0 --- /dev/null +++ b/src/research/study/agents/manifest_import.py @@ -0,0 +1,403 @@ +"""Import a *build* runtime manifest into a digest-pinned :class:`AgentReleaseV1`. + +The packaging pipeline emits a JSON build manifest (``manifest_version``, +``runtime_version``, ``server_commit``/``plugin_commit`` and ``artifacts[]``). +This module turns that document into the registry's own release contract so a +fresh database can be made immediately usable without a human hand-typing a +digest: + +* the release identity (``agent_id``/``release_id``/``version``) is derived + deterministically from the manifest, so re-importing identical bytes resolves + the *same* release instead of piling up duplicates; +* ``source_manifest_digest`` is the SHA-256 of the canonical manifest bytes; +* each built ``artifacts[]`` entry becomes a :class:`DistributionArtifact` whose + ``sha256`` is verified against the real archive **when the archive is present + under ``artifact_root``** -- a mismatch fails loudly rather than silently + trusting the declared digest; +* an artifact with a placeholder (all-zero) digest is a platform the build did + not actually produce. It is *excluded* and reported in ``skipped`` rather than + imported as a fake digest-pinned artifact. + +The module is deliberately free of database/App dependencies: the API router and +the local-dev seeder share exactly this planning step. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Optional + +from .enums import DistributionSourceType +from .models import ( + AdapterRef, + AgentReleaseV1, + DistributionArtifact, + ReleaseDisplay, +) + +__all__ = [ + "ManifestImportError", + "ManifestImportPlan", + "SkippedArtifact", + "build_manifest_release", + "canonical_manifest_bytes", + "manifest_digest", + "sha256_prefixed", +] + +#: Sentinel digest a build manifest uses for a platform it did not package. +PLACEHOLDER_DIGEST = "0" * 64 + + +class ManifestImportError(ValueError): + """A typed, operator-facing reason a build manifest cannot be imported. + + ``code``/``field`` mirror the registry's typed issue shape so the endpoint + can return a machine-readable rejection. + """ + + def __init__(self, code: str, message: str, field: str = "") -> None: + super().__init__(message) + self.code = code + self.message = message + self.field = field + + +@dataclass(frozen=True) +class SkippedArtifact: + """One ``artifacts[]`` entry that was not imported, and why.""" + + platform: str + archive: str + reason: str + + +@dataclass(frozen=True) +class ManifestImportPlan: + """The release a build manifest resolves to, plus what was excluded.""" + + release: AgentReleaseV1 + manifest_digest: str + skipped: list[SkippedArtifact] = field(default_factory=list) + + +def canonical_manifest_bytes(manifest: Mapping[str, Any]) -> bytes: + """Canonical UTF-8 bytes of a manifest mapping. + + Keys are sorted and separators are compact so the digest is stable for the + same semantic document regardless of formatting/whitespace: a re-import of + the "same" manifest is idempotent and never a new release. + """ + return json.dumps( + manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + + +def manifest_digest(manifest: Mapping[str, Any]) -> str: + """``sha256:`` of the canonical manifest bytes.""" + return sha256_prefixed(canonical_manifest_bytes(manifest)) + + +def sha256_prefixed(data: bytes) -> str: + """``sha256:`` for ``data`` (the repo-wide digest convention).""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def _bare_digest(value: Any) -> Optional[str]: + """Return the bare lowercase hex of a digest, or ``None`` if malformed.""" + text = str(value or "").strip().lower() + if text.startswith("sha256:"): + text = text[len("sha256:") :] + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + return None + return text + + +def _normalize_os(value: Any) -> str: + """Map a manifest platform name onto the repo's ``macos|linux|windows``.""" + text = str(value or "").strip().lower() + if text in {"macos", "darwin", "mac", "macosx"}: + return "macos" + if text.startswith("win"): + return "windows" + if text.startswith("linux"): + return "linux" + raise ManifestImportError( + "UNSUPPORTED_PLATFORM", f"unknown platform {value!r}", "platform" + ) + + +def _normalize_arch(value: Any) -> str: + """Map a manifest architecture onto the repo's ``arm64|x64`` vocabulary.""" + text = str(value or "").strip().lower() + if text in {"arm64", "aarch64"}: + return "arm64" + if text in {"x64", "x86_64", "amd64"}: + return "x64" + raise ManifestImportError( + "UNSUPPORTED_PLATFORM", f"unknown architecture {value!r}", "architecture" + ) + + +def _as_size(value: Any) -> Optional[int]: + """Coerce a size-like value to a positive int, or ``None``.""" + if isinstance(value, bool) or value is None: + return None + try: + size = int(value) + except (TypeError, ValueError): + return None + return size if size > 0 else None + + +def _resolve_archive(artifact_root: Optional[Path], archive: str) -> Optional[Path]: + """The on-disk path of an artifact archive, or ``None`` when unavailable.""" + if artifact_root is None or not archive: + return None + candidate = (artifact_root / archive).resolve() + if candidate.is_file(): + return candidate + basename = (artifact_root / Path(archive).name).resolve() + if basename.is_file(): + return basename + return None + + +def _file_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _size_from_supplied( + *, + archive: str, + platform: str, + raw_platform: str, + raw_arch: str, + sizes: Mapping[str, Any], +) -> Optional[int]: + """Look up a caller-supplied size by archive path / platform key / basename.""" + for key in ( + archive, + platform, + f"{raw_platform}-{raw_arch}", + Path(archive).name, + ): + if key and key in sizes: + resolved = _as_size(sizes[key]) + if resolved is not None: + return resolved + return None + + +def _derive_identity( + manifest: Mapping[str, Any], digest: str +) -> tuple[str, str, str]: + """Deterministically derive ``(agent_id, release_id, version)``.""" + artifacts = [item for item in manifest.get("artifacts") or [] if isinstance(item, Mapping)] + runtime_ids = sorted( + {str(item.get("runtime_id")).strip() for item in artifacts if item.get("runtime_id")} + ) + if len(runtime_ids) > 1: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest artifacts disagree on runtime_id: " + + ", ".join(runtime_ids), + "artifacts.runtime_id", + ) + agent_id = runtime_ids[0] if runtime_ids else "code4me-agent" + + version = str( + manifest.get("runtime_version") or manifest.get("version") or "" + ).strip() + if not version: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest declares no runtime_version", + "runtime_version", + ) + + # The digest is folded into the release id so a changed manifest is a + # *distinct* release, exactly as the registry's digest identity requires. + digest_hex = digest[len("sha256:") :] + release_id = f"{agent_id}-{version}-{digest_hex[:12]}" + return agent_id, release_id, version + + +def _derive_adapter( + manifest: Mapping[str, Any], *, agent_id: str, version: str, digest: str +) -> AdapterRef: + """The adapter identity for the release. + + A manifest may carry an explicit ``adapter``/``adapter_ref`` block; when it + does not, the packaging layer that emitted this manifest is identified by the + manifest digest itself (no digest is invented from thin air). + """ + explicit = manifest.get("adapter") or manifest.get("adapter_ref") + if isinstance(explicit, Mapping): + return AdapterRef.model_validate(dict(explicit)) + return AdapterRef( + adapter_id=f"{agent_id}-managed-adapter", + version=version, + digest=digest, + ) + + +def build_manifest_release( + manifest: Mapping[str, Any], + *, + artifact_root: Optional[str | Path] = None, + artifact_sizes: Optional[Mapping[str, Any]] = None, +) -> ManifestImportPlan: + """Build the :class:`AgentReleaseV1` a build manifest describes. + + ``artifact_root`` is the directory the manifest's relative ``archive`` paths + are resolved against. When a listed archive exists there its real digest and + size are computed and the declared digest is verified; when it does not, the + declared digest is trusted but an explicit size must be supplied (from the + manifest itself or ``artifact_sizes``) -- a size is never invented. + """ + if not isinstance(manifest, Mapping): + raise ManifestImportError( + "INVALID_MANIFEST", "manifest must be a JSON object", "manifest" + ) + raw_artifacts = manifest.get("artifacts") + if not isinstance(raw_artifacts, list) or not raw_artifacts: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest declares no artifacts", + "artifacts", + ) + + digest = manifest_digest(manifest) + agent_id, release_id, version = _derive_identity(manifest, digest) + root = Path(artifact_root) if artifact_root is not None else None + sizes = dict(artifact_sizes or {}) + + artifacts: list[DistributionArtifact] = [] + skipped: list[SkippedArtifact] = [] + seen: set[tuple[str, str]] = set() + + for index, raw in enumerate(raw_artifacts): + if not isinstance(raw, Mapping): + raise ManifestImportError( + "INVALID_MANIFEST", + "each artifacts[] entry must be an object", + f"artifacts[{index}]", + ) + raw_platform = raw.get("platform") + raw_arch = raw.get("architecture") + os_name = _normalize_os(raw_platform) + arch = _normalize_arch(raw_arch) + platform = f"{os_name}-{arch}" + if (os_name, arch) in seen: + raise ManifestImportError( + "INVALID_MANIFEST", + f"duplicate artifact for platform {platform!r}", + f"artifacts[{index}]", + ) + seen.add((os_name, arch)) + + archive = str(raw.get("archive") or "").strip() + declared = _bare_digest(raw.get("sha256")) + archive_path = _resolve_archive(root, archive) + + if declared is None or declared == PLACEHOLDER_DIGEST: + if archive_path is not None: + raise ManifestImportError( + "DIGEST_MISMATCH", + ( + f"artifact {archive!r} exists on disk but the manifest " + "declares a placeholder digest; a real artifact must " + "carry its real sha256" + ), + f"artifacts[{index}].sha256", + ) + # A placeholder digest is a platform the build did not produce. It is + # excluded explicitly instead of being imported as a fake pin. + skipped.append( + SkippedArtifact( + platform=platform, + archive=archive, + reason="manifest digest is a placeholder (platform not built)", + ) + ) + continue + + if archive_path is not None: + real = _file_digest(archive_path) + if real != declared: + raise ManifestImportError( + "DIGEST_MISMATCH", + ( + f"artifact {archive!r} digest {real} does not match the " + f"manifest's {declared}" + ), + f"artifacts[{index}].sha256", + ) + size = archive_path.stat().st_size + else: + size = _as_size(raw.get("size")) + if size is None: + size = _size_from_supplied( + archive=archive, + platform=platform, + raw_platform=str(raw_platform or ""), + raw_arch=str(raw_arch or ""), + sizes=sizes, + ) + if size is None: + raise ManifestImportError( + "SIZE_MISSING", + ( + f"artifact {archive or platform!r} is not available under " + "the artifact root and the manifest/caller supplies no " + "size; provide artifact_sizes for it" + ), + f"artifacts[{index}].size", + ) + + executable = raw.get("executable") + artifacts.append( + DistributionArtifact( + os=os_name, + arch=arch, + path=archive or f"code4me-runtime/{platform}", + sha256="sha256:" + declared, + size=size, + executable=str(executable).strip() if executable else None, + ) + ) + + if not artifacts: + raise ManifestImportError( + "INCOMPLETE_ARTIFACTS", + "manifest has no built artifact with a real digest", + "artifacts", + ) + + release = AgentReleaseV1( + agent_id=agent_id, + release_id=release_id, + version=version, + display=ReleaseDisplay( + name=f"Code4Me agent runtime {version}", + vendor="code4me2", + description="Imported from the built runtime manifest.", + ), + source_type=DistributionSourceType.BUNDLED, + source_manifest_digest=digest, + artifacts=artifacts, + adapter=_derive_adapter( + manifest, agent_id=agent_id, version=version, digest=digest + ), + created_at=datetime.now(timezone.utc), + ) + return ManifestImportPlan(release=release, manifest_digest=digest, skipped=skipped) diff --git a/src/research/study/agents/models.py b/src/research/study/agents/models.py new file mode 100644 index 0000000..5bdf1b0 --- /dev/null +++ b/src/research/study/agents/models.py @@ -0,0 +1,282 @@ +"""Pydantic v2 contracts for the agent registry and capability contract. + +Two documents are defined: + +* :class:`AgentReleaseV1` - the immutable distribution/release identity a study + condition pins. Its digest identity is + ``(agent_id, release_id, source_manifest_digest)``: the same agent version + with a different source manifest digest is a *distinct* release, never an + overwrite. +* :class:`CapabilitySnapshotV1` - per-run declared-vs-observed capability + evidence. ``declared`` and ``observed`` are kept in separate maps so a + declared capability that was never observed stays ``DECLARED`` with an + explicit ``UNAVAILABLE``/``UNKNOWN`` observation. A missing measurement is + ``value is None`` plus a coverage state, never ``0``, ``False`` or success. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.compatibility.enums import Fidelity + +from .enums import ( + CapabilityCoverageState, + DistributionMode, + DistributionSourceType, + QualificationStatus, + SnapshotCapabilityState, +) + +_BASE_CONFIG = ConfigDict(extra="forbid") +_FROZEN_CONFIG = ConfigDict(extra="forbid", frozen=True) + +_AARCH64_ARCHS = frozenset({"aarch64", "arm64"}) +_X64_ARCHS = frozenset({"x86_64", "amd64", "x64"}) + + +def normalize_platform(os_name: str, arch: str) -> tuple[str, str]: + """Canonicalise an ``(os, arch)`` pair for platform comparison. + + Hosts report the same platform under different names (``Darwin`` vs + ``macos``, ``AMD64`` vs ``x64``), so the requested tuple and every stored + artifact tuple are normalised before comparison instead of blocking on a + cosmetic string mismatch. + """ + lowered_os = (os_name or "").lower() + if "mac" in lowered_os or lowered_os == "darwin": + canonical_os = "macos" + elif lowered_os.startswith("win"): + canonical_os = "windows" + elif "linux" in lowered_os: + canonical_os = "linux" + else: + canonical_os = lowered_os + + lowered_arch = (arch or "").lower() + if lowered_arch in _AARCH64_ARCHS: + canonical_arch = "aarch64" + elif lowered_arch in _X64_ARCHS: + canonical_arch = "x64" + else: + canonical_arch = lowered_arch + + return (canonical_os, canonical_arch) + + +class ReleaseDisplay(BaseModel): + """Human-facing, non-secret release metadata.""" + + model_config = _BASE_CONFIG + + name: Optional[str] = None + vendor: Optional[str] = None + description: Optional[str] = None + homepage: Optional[str] = None + + +class DistributionArtifact(BaseModel): + """One platform-specific distribution artifact of a release. + + ``path`` is deliberately *relative*: releases must never carry a local + absolute path or host-specific mutable registry configuration. + """ + + model_config = _BASE_CONFIG + + os: str + arch: str + path: str + sha256: str + size: int + executable: Optional[str] = None + license: Optional[str] = None + license_review: Optional[str] = None + signature: Optional[str] = None + + @property + def platform(self) -> tuple[str, str]: + """The ``(os, arch)`` platform this artifact targets.""" + return (self.os, self.arch) + + +class AdapterRef(BaseModel): + """Identity of the adapter/packaging layer that drives one release.""" + + model_config = _BASE_CONFIG + + adapter_id: str + version: str + digest: Optional[str] = None + # Declared ranges (e.g. ``">=1.2.0,<2.0.0"``) the adapter claims to drive. + supported_release_ranges: list[str] = Field(default_factory=list) + + +class AgentReleaseV1(BaseModel): + """An immutable, digest-pinned agent release record (schema version 1). + + ``distribution_mode`` selects the distribution contract: ``PACKAGED`` + (default) requires at least one digest-pinned artifact, while + ``BYOA_EXTERNAL`` (participant-installed) requires a command/package identity + and carries no artifact digest. The mode is additive so already-stored + releases without the field continue to parse as ``PACKAGED``. + """ + + model_config = _BASE_CONFIG + + schema_version: str = "1" + agent_id: str + # Stable, unique release identifier. It is part of the digest identity and + # must be changed when the upstream manifest digest changes. + release_id: str + version: str + display: ReleaseDisplay = Field(default_factory=ReleaseDisplay) + source_type: DistributionSourceType = DistributionSourceType.RESEARCH_OVERLAY + source_manifest_digest: str + distribution_mode: DistributionMode = DistributionMode.PACKAGED + artifacts: list[DistributionArtifact] = Field(default_factory=list) + # BYOA only: the explicit participant-installed agent command (an executable + # name or a portable relative reference; never a local absolute path). A + # participant host may override it with a configured path. + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + # BYOA only: the logical package (``goose``/``codex``/...) used for discovery + # when no explicit command is pinned. + agent_package: Optional[str] = None + adapter: Optional[AdapterRef] = None + # Negotiated ACP protocol range the release is compatible with. + min_protocol_version: Optional[str] = None + max_protocol_version: Optional[str] = None + # Derived server-side from verified conformance evidence + # (``release_json.conformance[]``); callers can never supply it. Absent a + # passing receipt a release is ``UNQUALIFIED``. + qualification_status: QualificationStatus = QualificationStatus.UNQUALIFIED + created_at: Optional[datetime] = None + + @property + def digest_identity(self) -> tuple[str, str, str]: + """The identity that makes two release records the same release.""" + return (self.agent_id, self.release_id, self.source_manifest_digest) + + @property + def is_byoa(self) -> bool: + """True when the agent is participant-installed rather than packaged.""" + return self.distribution_mode == DistributionMode.BYOA_EXTERNAL + + @property + def byoa_identity(self) -> Optional[str]: + """The BYOA command/package identity, or ``None`` for a packaged release.""" + if not self.is_byoa: + return None + return ( + (self.agent_command or "").strip() + or (self.agent_package or "").strip() + or None + ) + + def artifact_for(self, os_name: str, arch: str) -> Optional[DistributionArtifact]: + """Return the exact artifact for ``(os_name, arch)`` or ``None``. + + There is intentionally no fallback: an unavailable platform must block + before download/launch rather than silently selecting another artifact. + """ + requested = normalize_platform(os_name, arch) + for artifact in self.artifacts: + if normalize_platform(artifact.os, artifact.arch) == requested: + return artifact + return None + + +class CapabilityEntry(BaseModel): + """One capability measurement or declaration. + + ``value`` is nullable and is *not* defaulted: a missing measurement is + ``None`` and its meaning is carried by ``state``, the fidelity and the + limitations. It is never coerced to ``0``, ``False`` or success. + """ + + model_config = _FROZEN_CONFIG + + state: SnapshotCapabilityState + value: Optional[Any] = None + fidelity: Fidelity = Fidelity.NORMALIZED + source: str = "handshake" + limitations: list[str] = Field(default_factory=list) + observed_at: Optional[datetime] = None + + +class EnvironmentRef(BaseModel): + """Opaque reference to the environment a snapshot was captured against.""" + + model_config = _BASE_CONFIG + + environment_id: Optional[str] = None + host_kind: Optional[str] = None + os: Optional[str] = None + arch: Optional[str] = None + ide_build: Optional[str] = None + plugin_version: Optional[str] = None + + +class SnapshotFailure(BaseModel): + """Failure captured while producing a snapshot, if any.""" + + model_config = _BASE_CONFIG + + reason: str + detail: Optional[str] = None + occurred_at: Optional[datetime] = None + + +class CapabilitySnapshotV1(BaseModel): + """Immutable declared-vs-observed capability snapshot (schema version 1).""" + + model_config = _FROZEN_CONFIG + + schema_version: str = "1" + snapshot_id: UUID + release_id: str + agent_id: str + adapter_id: Optional[str] = None + adapter_version: Optional[str] = None + environment: EnvironmentRef = Field(default_factory=EnvironmentRef) + protocol_version: str + # Declared and observed are separate maps keyed by capability name. Unknown + # capability names are preserved rather than dropped. + declared: dict[str, CapabilityEntry] = Field(default_factory=dict) + observed: dict[str, CapabilityEntry] = Field(default_factory=dict) + evidence_refs: list[str] = Field(default_factory=list) + captured_at: datetime + failure: Optional[SnapshotFailure] = None + + +class CapabilityCoverage(BaseModel): + """Declared-vs-observed coverage for one capability. + + Both sides keep their explicit state (including ``UNKNOWN`` and + ``UNAVAILABLE``); the summary never collapses them into a boolean. + """ + + model_config = _BASE_CONFIG + + capability: str + declared_state: SnapshotCapabilityState = SnapshotCapabilityState.UNKNOWN + observed_state: SnapshotCapabilityState = SnapshotCapabilityState.UNKNOWN + coverage: CapabilityCoverageState + value_present: bool = False + limitations: list[str] = Field(default_factory=list) + + +class CapabilityCoverageReport(BaseModel): + """Coverage summary for one snapshot, with explicit state counts.""" + + model_config = _BASE_CONFIG + + snapshot_id: UUID + release_id: str + entries: list[CapabilityCoverage] = Field(default_factory=list) + counts: dict[str, int] = Field(default_factory=dict) diff --git a/src/research/study/agents/registry.py b/src/research/study/agents/registry.py new file mode 100644 index 0000000..565d6fa --- /dev/null +++ b/src/research/study/agents/registry.py @@ -0,0 +1,784 @@ +"""Pure agent-registry service: releases, qualification and capability evidence. + +This module has no database or application dependency. It operates on the +Pydantic contracts in :mod:`research.study.agents.models` and returns typed results so +an API can render field-level errors and an audit can record why an operation +was blocked. + +Key invariants: + +* A release's identity is ``(agent_id, release_id, source_manifest_digest)``. A + changed digest requires a distinct release record; an exact duplicate is + rejected with ``DUPLICATE_RELEASE``. +* Qualification is **derived** from verified conformance evidence (a passing + receipt recorded by the packaging layer), never supplied by a caller. There is + no reviewer transition that can mark a release qualified. +* Platform resolution is exact and never falls back to another platform. +* Declared and observed capabilities are stored in separate maps and unknown or + unavailable measurements stay visible (``value`` stays ``None``). +""" + +from __future__ import annotations + +import re +import uuid +from datetime import datetime, timezone +from typing import Any, Mapping, Optional, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + CapabilityCoverageState, + DistributionMode, + QualificationStatus, + RegistryReasonCode, + SnapshotCapabilityState, +) +from .models import ( + AdapterRef, + AgentReleaseV1, + CapabilityCoverage, + CapabilityCoverageReport, + CapabilityEntry, + CapabilitySnapshotV1, + DistributionArtifact, + EnvironmentRef, + SnapshotFailure, +) + +_BASE_CONFIG = ConfigDict(extra="forbid") + +#: Conformance receipt status that backs a derived ``QUALIFIED``. +_PASSED_CONFORMANCE = "PASS" + + +def _normalize_digest(value: Any) -> Optional[str]: + text = str(value or "").strip().lower() + if not text: + return None + return text[7:] if text.startswith("sha256:") else text + + +def _same_digest(left: Any, right: Any) -> bool: + normalized = _normalize_digest(left) + return normalized is not None and normalized == _normalize_digest(right) + +# Qualification states that may be selected by a study condition. +SELECTABLE_STATUSES = frozenset({QualificationStatus.QUALIFIED}) + +EntryInput = Union[CapabilityEntry, SnapshotCapabilityState, str, None] + + +class RegistryIssue(BaseModel): + """One typed registry rejection reason.""" + + model_config = _BASE_CONFIG + + code: RegistryReasonCode + message: str + field: str = "" + + +class RegistrationResult(BaseModel): + """Outcome of registering a release.""" + + model_config = _BASE_CONFIG + + accepted: bool + release: Optional[AgentReleaseV1] = None + issue: Optional[RegistryIssue] = None + + +class ArtifactResolution(BaseModel): + """Outcome of selecting a distribution artifact for a platform.""" + + model_config = _BASE_CONFIG + + resolved: bool + artifact: Optional[DistributionArtifact] = None + issue: Optional[RegistryIssue] = None + + +class DistributionResolution(BaseModel): + """Outcome of resolving the distribution contract for one platform. + + A ``PACKAGED`` release resolves to a digest-pinned [artifact]; a + ``BYOA_EXTERNAL`` release resolves to the participant-installable + command/package identity and carries no artifact. + """ + + model_config = _BASE_CONFIG + + resolved: bool + distribution_mode: DistributionMode = DistributionMode.PACKAGED + artifact: Optional[DistributionArtifact] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + agent_package: Optional[str] = None + issue: Optional[RegistryIssue] = None + + +class QualificationAssessment(BaseModel): + """Whether a release currently satisfies the qualification contract.""" + + model_config = _BASE_CONFIG + + qualifiable: bool + blockers: list[RegistryIssue] = Field(default_factory=list) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue(code: RegistryReasonCode, message: str, field: str = "") -> RegistryIssue: + return RegistryIssue(code=code, message=message, field=field) + + +def _distribution_issue(release: AgentReleaseV1) -> Optional[RegistryIssue]: + """Validate the mode-specific distribution contract, or ``None`` if valid. + + ``PACKAGED`` requires at least one artifact with a non-empty digest; + ``BYOA_EXTERNAL`` requires a command/package identity and is *not* expected + to carry a digest. The check is shared by registration and qualification so + a release can never be accepted under one contract and qualified under another. + """ + if release.is_byoa: + if release.byoa_identity is None: + return _issue( + RegistryReasonCode.AGENT_NOT_FOUND, + "a BYOA release must declare an agent command or an agent package", + "agent_command", + ) + return None + + if not release.artifacts: + return _issue( + RegistryReasonCode.ARTIFACT_MISSING, + "a PACKAGED release must publish at least one distribution artifact", + "artifacts", + ) + for index, artifact in enumerate(release.artifacts): + if not artifact.sha256.strip(): + return _issue( + RegistryReasonCode.DIGEST_MISMATCH, + "a PACKAGED distribution artifact must carry a sha256 digest", + f"artifacts[{index}].sha256", + ) + return None + + +def derive_qualification_status( + release_json: Optional[Mapping[str, Any]], +) -> QualificationStatus: + """Derive a release's qualification from verified conformance evidence. + + Artifact identity and conformance receipts are owned by the packaging layer + and persisted alongside the release in ``agent_release.release_json``. A + release is ``QUALIFIED`` only when at least one recorded conformance receipt + for it (a) has status ``PASS``, (b) binds to the release's exact artifact + identity — artifact digest, adapter digest and OS/architecture — and (c) + contains at least one passing case. Anything else (no receipt, a failing + receipt, a receipt bound to a different artifact/adapter/platform, a + passing receipt with no case results, or a caller-supplied status) is + ``UNQUALIFIED``. A synthetic fixture or any unrelated ``PASS`` receipt can + therefore never qualify a real runtime. + """ + if not isinstance(release_json, Mapping): + return QualificationStatus.UNQUALIFIED + receipts = release_json.get("conformance") or [] + if not isinstance(receipts, (list, tuple)): + return QualificationStatus.UNQUALIFIED + for receipt in receipts: + if not isinstance(receipt, Mapping): + continue + if str(receipt.get("status", "")).strip().upper() != _PASSED_CONFORMANCE: + continue + if not _receipt_binds_to_release(receipt, release_json): + continue + if not _receipt_has_passing_case(receipt): + continue + return QualificationStatus.QUALIFIED + return QualificationStatus.UNQUALIFIED + + +def _receipt_binds_to_release( + receipt: Mapping[str, Any], release_json: Mapping[str, Any] +) -> bool: + """Whether a PASS receipt is bound to this exact artifact/adapter/platform.""" + artifact_digest = str(receipt.get("artifact_digest") or "").strip().lower() + if not artifact_digest: + return False + + artifacts = [a for a in (release_json.get("artifacts") or []) if isinstance(a, Mapping)] + components = [ + c + for c in ((release_json.get("package_json") or {}).get("components") or []) + if isinstance(c, Mapping) + ] + matches_artifact = any( + _same_digest(item.get("sha256"), artifact_digest) + for item in (*artifacts, *components) + ) + matches_byoa = not artifacts and _same_digest( + release_json.get("source_manifest_digest"), artifact_digest + ) + if not (matches_artifact or matches_byoa): + return False + + adapter = release_json.get("adapter") + if isinstance(adapter, Mapping) and adapter.get("digest"): + if not _same_digest(adapter.get("digest"), receipt.get("adapter_digest")): + return False + + host = receipt.get("host") + if isinstance(host, Mapping) and host.get("os") and host.get("arch"): + platforms = [ + {"os": a.get("os"), "arch": a.get("arch")} + for a in artifacts + if a.get("os") and a.get("arch") + ] + manifest = release_json.get("package_json") or {} + platforms += [ + {"os": p.get("os"), "arch": p.get("arch")} + for p in (manifest.get("supported_platforms") or []) + if isinstance(p, Mapping) and p.get("os") and p.get("arch") + ] + if platforms and not any( + str(p.get("os")).lower() == str(host.get("os")).lower() + and str(p.get("arch")).lower() == str(host.get("arch")).lower() + for p in platforms + ): + return False + return True + + +def _receipt_has_passing_case(receipt: Mapping[str, Any]) -> bool: + """A PASS receipt with no observed case results is not execution evidence.""" + cases = receipt.get("case_results") + if not isinstance(cases, (list, tuple)) or not cases: + return False + return any( + isinstance(case, Mapping) + and str(case.get("status", "")).strip().upper() == _PASSED_CONFORMANCE + for case in cases + ) + + +# --------------------------------------------------------------------------- +# Per-release verified approval options (phase 04; folded from approvals.py) +# --------------------------------------------------------------------------- +# +# An agent profile selects an approval policy, but a study may only require an +# option the selected release's conformance evidence actually exercises. The +# baseline ``auto`` policy needs no host permission gate, so it is always +# available. ``per_step`` (ask per turn) requires a passing permission-request +# case; ``suggestion_only`` (propose edits without applying) requires a passing +# edit-proposal/diff case. Absent evidence, the gated options are **not** offered +# — a matching version string is never enough. The ``auto``/``per_step``/ +# ``suggestion_only`` vocabulary has one owner: ``backend.routers.agent.profiles`` +# validates the same values. + +APPROVAL_AUTO = "auto" +APPROVAL_PER_STEP = "per_step" +APPROVAL_SUGGESTION_ONLY = "suggestion_only" + +ALL_APPROVAL_OPTIONS: tuple[str, ...] = ( + APPROVAL_AUTO, + APPROVAL_PER_STEP, + APPROVAL_SUGGESTION_ONLY, +) + +#: Substrings of a passing conformance case id that verify each gated option. +_PER_STEP_CASE_HINTS = ("permission",) +_SUGGESTION_CASE_HINTS = ("edit", "suggestion", "diff") + + +def _passed_approval_case_ids(release_json: Optional[Mapping[str, Any]]) -> set[str]: + if not isinstance(release_json, Mapping): + return set() + receipts = release_json.get("conformance") or [] + if not isinstance(receipts, (list, tuple)): + return set() + passed: set[str] = set() + for receipt in receipts: + if not isinstance(receipt, Mapping): + continue + if str(receipt.get("status", "")).strip().upper() != _PASSED_CONFORMANCE: + continue + cases = receipt.get("case_results") or [] + if not isinstance(cases, (list, tuple)): + continue + for case in cases: + if not isinstance(case, Mapping): + continue + if str(case.get("status", "")).strip().upper() != _PASSED_CONFORMANCE: + continue + case_id = str(case.get("case_id", "")).strip() + if case_id: + passed.add(case_id) + return passed + + +def verified_approval_options(release_json: Optional[Mapping[str, Any]]) -> list[str]: + """Return the approval options the release's evidence verifies. + + Always includes ``auto``. ``per_step`` and ``suggestion_only`` are included + only when a passing conformance case exercises the corresponding host + behaviour, so an unsupported option is never offered to the editor. + """ + passed = {case_id.lower() for case_id in _passed_approval_case_ids(release_json)} + options = [APPROVAL_AUTO] + if any(hint in case_id for case_id in passed for hint in _PER_STEP_CASE_HINTS): + options.append(APPROVAL_PER_STEP) + if any(hint in case_id for case_id in passed for hint in _SUGGESTION_CASE_HINTS): + options.append(APPROVAL_SUGGESTION_ONLY) + return options + + +def approval_option_verified( + release_json: Optional[Mapping[str, Any]], option: str +) -> bool: + """Whether ``option`` is verified for the release described by ``release_json``.""" + return (option or "").strip().lower() in verified_approval_options(release_json) + + +def _coerce_entry(value: EntryInput, default_state: SnapshotCapabilityState) -> CapabilityEntry: + """Normalize a capability map value into a :class:`CapabilityEntry`. + + ``None`` becomes an entry with the default state and ``value=None``: a + missing measurement is never turned into ``0`` or ``False``. + """ + if isinstance(value, CapabilityEntry): + return value + if isinstance(value, SnapshotCapabilityState): + return CapabilityEntry(state=value, value=None) + if isinstance(value, str): + try: + state = SnapshotCapabilityState(value.strip().upper()) + except ValueError: + state = SnapshotCapabilityState.UNKNOWN + return CapabilityEntry(state=state, value=None) + if value is None: + return CapabilityEntry(state=default_state, value=None) + return CapabilityEntry(state=default_state, value=value) + + +def build_capability_snapshot( + snapshot_id: UUID, + release: AgentReleaseV1, + *, + protocol_version: str, + declared: Mapping[str, EntryInput], + observed: Mapping[str, EntryInput], + environment: Optional[EnvironmentRef] = None, + adapter_id: Optional[str] = None, + adapter_version: Optional[str] = None, + evidence_refs: Optional[list[str]] = None, + captured_at: Optional[datetime] = None, + failure: Optional[SnapshotFailure] = None, +) -> CapabilitySnapshotV1: + """Build an immutable snapshot from a declared/observed observation pair. + + The two maps are preserved independently: a capability present only in + ``declared`` is not synthesized into ``observed``, and vice versa. + """ + return CapabilitySnapshotV1( + snapshot_id=snapshot_id, + release_id=release.release_id, + agent_id=release.agent_id, + adapter_id=adapter_id or (release.adapter.adapter_id if release.adapter else None), + adapter_version=adapter_version + or (release.adapter.version if release.adapter else None), + environment=environment or EnvironmentRef(), + protocol_version=protocol_version, + declared={ + capability: _coerce_entry(value, SnapshotCapabilityState.DECLARED) + for capability, value in declared.items() + }, + observed={ + capability: _coerce_entry(value, SnapshotCapabilityState.UNKNOWN) + for capability, value in observed.items() + }, + evidence_refs=list(evidence_refs or []), + captured_at=captured_at or _now(), + failure=failure, + ) + + +def _coverage_state( + declared_state: SnapshotCapabilityState, observed_state: SnapshotCapabilityState +) -> CapabilityCoverageState: + if ( + declared_state == SnapshotCapabilityState.BROKEN + or observed_state == SnapshotCapabilityState.BROKEN + ): + return CapabilityCoverageState.BROKEN + if observed_state == SnapshotCapabilityState.PARTIAL: + return CapabilityCoverageState.PARTIAL + if observed_state == SnapshotCapabilityState.UNAVAILABLE: + return CapabilityCoverageState.UNAVAILABLE + if observed_state == SnapshotCapabilityState.OBSERVED: + return CapabilityCoverageState.OBSERVED + if declared_state in ( + SnapshotCapabilityState.DECLARED, + SnapshotCapabilityState.OBSERVED, + ): + return CapabilityCoverageState.DECLARED_ONLY + return CapabilityCoverageState.UNKNOWN + + +def capability_coverage(snapshot: CapabilitySnapshotV1) -> list[CapabilityCoverage]: + """Summarize declared vs observed state per capability (never a boolean).""" + capabilities = sorted(set(snapshot.declared) | set(snapshot.observed)) + entries: list[CapabilityCoverage] = [] + for capability in capabilities: + declared = snapshot.declared.get(capability) + observed = snapshot.observed.get(capability) + declared_state = ( + declared.state if declared else SnapshotCapabilityState.UNKNOWN + ) + observed_state = ( + observed.state if observed else SnapshotCapabilityState.UNKNOWN + ) + limitations: list[str] = [] + if declared: + limitations.extend(declared.limitations) + if observed: + limitations.extend(observed.limitations) + entries.append( + CapabilityCoverage( + capability=capability, + declared_state=declared_state, + observed_state=observed_state, + coverage=_coverage_state(declared_state, observed_state), + value_present=observed is not None and observed.value is not None, + limitations=limitations, + ) + ) + return entries + + +def coverage_report(snapshot: CapabilitySnapshotV1) -> CapabilityCoverageReport: + """Build the coverage report (entries + explicit state counts).""" + entries = capability_coverage(snapshot) + counts: dict[str, int] = {} + for entry in entries: + counts[entry.coverage.value] = counts.get(entry.coverage.value, 0) + 1 + return CapabilityCoverageReport( + snapshot_id=snapshot.snapshot_id, + release_id=snapshot.release_id, + entries=entries, + counts=counts, + ) + + +class AgentRegistry: + """In-memory registry of releases and capability snapshots. + + The service is deliberately storage-agnostic. The API layer loads rows from + the database into a registry instance (or wraps it in + :class:`research.study.agents.resolver.RegistryReleaseResolver`) and persists the + results through :mod:`research.study.agents.store`. + """ + + def __init__(self) -> None: + self._by_identity: dict[tuple[str, str, str], AgentReleaseV1] = {} + self._by_release_id: dict[tuple[str, str], AgentReleaseV1] = {} + self._by_version: dict[tuple[str, str], list[AgentReleaseV1]] = {} + self._snapshots: dict[UUID, CapabilitySnapshotV1] = {} + + # -- releases --------------------------------------------------------- + + def register_release(self, release: AgentReleaseV1) -> RegistrationResult: + """Register a release, rejecting duplicate digest identities.""" + if not release.source_manifest_digest.strip(): + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DIGEST_MISMATCH, + "source_manifest_digest must be non-empty", + "source_manifest_digest", + ), + ) + + distribution_issue = _distribution_issue(release) + if distribution_issue is not None: + return RegistrationResult(accepted=False, issue=distribution_issue) + + identity = release.digest_identity + if identity in self._by_identity: + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DUPLICATE_RELEASE, + "an identical release digest already exists", + "source_manifest_digest", + ), + ) + + key = (release.agent_id, release.release_id) + existing = self._by_release_id.get(key) + if existing is not None: + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DUPLICATE_RELEASE, + ( + "release_id is already registered for this agent with a " + "different digest; a changed digest requires a new release_id" + ), + "release_id", + ), + ) + + stored = release.model_copy(deep=True) + self._by_identity[identity] = stored + self._by_release_id[key] = stored + self._by_version.setdefault((release.agent_id, release.version), []).append( + stored + ) + return RegistrationResult(accepted=True, release=stored.model_copy(deep=True)) + + def get_release( + self, agent_id: str, release_id: str + ) -> Optional[AgentReleaseV1]: + """Fetch a release by its natural ``(agent_id, release_id)`` key.""" + return self._by_release_id.get((agent_id, release_id)) + + def find_release( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> Optional[AgentReleaseV1]: + """Resolve a release by explicit id, or by a unique version label.""" + if release_id: + return self._by_release_id.get((agent_id, release_id)) + if version: + candidates = self._by_version.get((agent_id, version), []) + if len(candidates) == 1: + return candidates[0] + return None + + def version_candidates( + self, agent_id: str, version: str + ) -> list[AgentReleaseV1]: + """All releases registered for ``(agent_id, version)``.""" + return list(self._by_version.get((agent_id, version), [])) + + def list_releases(self, agent_id: Optional[str] = None) -> list[AgentReleaseV1]: + """List releases, optionally filtered by agent id, in a stable order.""" + releases = list(self._by_release_id.values()) + if agent_id is not None: + releases = [release for release in releases if release.agent_id == agent_id] + return sorted( + releases, + key=lambda release: (release.agent_id, release.version, release.release_id), + ) + + # -- platform selection / qualification ------------------------------ + + def resolve_artifact( + self, release: AgentReleaseV1, os_name: str, arch: str + ) -> ArtifactResolution: + """Return the exact artifact for ``(os_name, arch)`` with no fallback.""" + if not release.artifacts: + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.ARTIFACT_MISSING, + "release publishes no distribution artifacts", + "artifacts", + ), + ) + + artifact = release.artifact_for(os_name, arch) + if artifact is None: + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.UNSUPPORTED_PLATFORM, + f"no artifact for platform ({os_name}, {arch})", + "artifacts", + ), + ) + if not artifact.sha256.strip(): + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.DIGEST_MISMATCH, + f"artifact for ({os_name}, {arch}) has no digest", + "artifacts", + ), + ) + return ArtifactResolution(resolved=True, artifact=artifact) + + def resolve_distribution( + self, release: AgentReleaseV1, os_name: str, arch: str + ) -> DistributionResolution: + """Resolve the mode-specific distribution contract with no fallback. + + A ``BYOA_EXTERNAL`` release resolves to its command/package identity (no + artifact); a ``PACKAGED`` release delegates to :meth:`resolve_artifact` so + a missing/undigested artifact still blocks. + """ + if release.is_byoa: + if release.byoa_identity is None: + return DistributionResolution( + resolved=False, + distribution_mode=DistributionMode.BYOA_EXTERNAL, + issue=_issue( + RegistryReasonCode.AGENT_NOT_FOUND, + "BYOA release declares neither a command nor an agent package", + "agent_command", + ), + ) + return DistributionResolution( + resolved=True, + distribution_mode=DistributionMode.BYOA_EXTERNAL, + agent_command=release.agent_command, + agent_command_args=list(release.agent_command_args), + agent_package=release.agent_package, + ) + + artifact_resolution = self.resolve_artifact(release, os_name, arch) + if not artifact_resolution.resolved: + return DistributionResolution( + resolved=False, + distribution_mode=DistributionMode.PACKAGED, + issue=artifact_resolution.issue, + ) + return DistributionResolution( + resolved=True, + distribution_mode=DistributionMode.PACKAGED, + artifact=artifact_resolution.artifact, + ) + + def assess_qualification( + self, release: AgentReleaseV1 + ) -> QualificationAssessment: + """Return the blockers preventing a release from being qualified.""" + blockers: list[RegistryIssue] = [] + + distribution_issue = _distribution_issue(release) + if distribution_issue is not None: + blockers.append(distribution_issue) + elif not release.is_byoa: + for index, artifact in enumerate(release.artifacts): + if not artifact.path.strip(): + blockers.append( + _issue( + RegistryReasonCode.ARTIFACT_MISSING, + "distribution artifact path must be non-empty", + f"artifacts[{index}].path", + ) + ) + + adapter: Optional[AdapterRef] = release.adapter + if adapter is None: + blockers.append( + _issue( + RegistryReasonCode.ADAPTER_INCOMPATIBLE, + "a qualified release must declare an adapter", + "adapter", + ) + ) + else: + if not adapter.version.strip(): + blockers.append( + _issue( + RegistryReasonCode.ADAPTER_INCOMPATIBLE, + "adapter version must be non-empty", + "adapter.version", + ) + ) + if not (adapter.digest or "").strip(): + blockers.append( + _issue( + RegistryReasonCode.ADAPTER_INCOMPATIBLE, + "adapter digest must be non-empty", + "adapter.digest", + ) + ) + + minimum = release.min_protocol_version + maximum = release.max_protocol_version + if minimum and maximum and _version_tuple(minimum) > _version_tuple(maximum): + blockers.append( + _issue( + RegistryReasonCode.INVALID_VERSION_RANGE, + f"max_protocol_version {maximum!r} is below {minimum!r}", + "max_protocol_version", + ) + ) + + return QualificationAssessment(qualifiable=not blockers, blockers=blockers) + + # -- snapshots -------------------------------------------------------- + + def record_snapshot( + self, + release: AgentReleaseV1, + *, + protocol_version: str, + declared: Mapping[str, EntryInput], + observed: Mapping[str, EntryInput], + environment: Optional[EnvironmentRef] = None, + adapter_id: Optional[str] = None, + adapter_version: Optional[str] = None, + evidence_refs: Optional[list[str]] = None, + captured_at: Optional[datetime] = None, + failure: Optional[SnapshotFailure] = None, + ) -> CapabilitySnapshotV1: + """Build and store an immutable capability snapshot for ``release``.""" + snapshot = build_capability_snapshot( + uuid.uuid4(), + release, + protocol_version=protocol_version, + declared=declared, + observed=observed, + environment=environment, + adapter_id=adapter_id, + adapter_version=adapter_version, + evidence_refs=evidence_refs, + captured_at=captured_at, + failure=failure, + ) + self.add_snapshot(snapshot) + return snapshot + + def add_snapshot(self, snapshot: CapabilitySnapshotV1) -> CapabilitySnapshotV1: + """Store an already-built snapshot (e.g. rehydrated from storage).""" + self._snapshots[snapshot.snapshot_id] = snapshot + return snapshot + + def get_snapshot(self, snapshot_id: UUID) -> Optional[CapabilitySnapshotV1]: + """Fetch a snapshot by id, or ``None``.""" + return self._snapshots.get(snapshot_id) + + def list_snapshots( + self, release_id: Optional[str] = None + ) -> list[CapabilitySnapshotV1]: + """List snapshots, optionally for one release, oldest-first.""" + snapshots = list(self._snapshots.values()) + if release_id is not None: + snapshots = [ + snapshot + for snapshot in snapshots + if snapshot.release_id == release_id + ] + return sorted(snapshots, key=lambda snapshot: snapshot.captured_at) + + def coverage(self, snapshot_id: UUID) -> Optional[CapabilityCoverageReport]: + """Coverage report for one stored snapshot, or ``None`` if unknown.""" + snapshot = self.get_snapshot(snapshot_id) + if snapshot is None: + return None + return coverage_report(snapshot) + + +def _version_tuple(value: str) -> tuple[int, ...]: + return tuple(int(part) for part in re.findall(r"\d+", value)) diff --git a/src/research/study/agents/resolver.py b/src/research/study/agents/resolver.py new file mode 100644 index 0000000..623395c --- /dev/null +++ b/src/research/study/agents/resolver.py @@ -0,0 +1,172 @@ +"""Bridge from the agent registry to protocol release validation. + +:class:`RegistryReleaseResolver` implements the +:class:`research.study.protocol.validation.ReleaseResolver` interface so protocol +publication rejects releases that are missing, retired, blocked, or not yet +qualified: + +* not found -> ``NOT_FOUND`` +* ``RETIRED`` / ``BLOCKED`` -> ``WITHDRAWN`` +* ``UNQUALIFIED`` / ``DRAFT`` / ``CONDITIONALLY_QUALIFIED`` -> ``UNQUALIFIED`` +* ``QUALIFIED`` -> ``RESOLVED`` with the selected artifact's digest +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from research.study.protocol.enums import ReleaseResolutionStatus +from research.study.protocol.validation import ReleaseResolution + +from .enums import QualificationStatus + +if TYPE_CHECKING: + from .models import AgentReleaseV1 + from .registry import AgentRegistry + +_WITHDRAWN = frozenset( + {QualificationStatus.RETIRED, QualificationStatus.BLOCKED} +) +_UNQUALIFIED = frozenset( + { + QualificationStatus.UNQUALIFIED, + QualificationStatus.DRAFT, + QualificationStatus.CONDITIONALLY_QUALIFIED, + } +) + + +class RegistryReleaseResolver: + """Resolve a study condition's pinned release against the registry.""" + + def __init__( + self, + registry: AgentRegistry, + *, + platform: Optional[tuple[str, str]] = None, + ) -> None: + self.registry = registry + self.platform = platform + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: + """Map a release reference onto a typed protocol resolution.""" + if version is not None and version.strip().lower() == "latest": + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message="'latest' is a mutable reference and cannot be pinned", + ) + + release = self.registry.find_release( + agent_id, release_id=release_id, version=version + ) + if release is None: + if version is not None and self.registry.version_candidates( + agent_id, version + ): + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message=( + "version label is ambiguous; pin an explicit release_id" + ), + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.NOT_FOUND, + agent_id=agent_id, + release_id=release_id, + version=version, + message="release is not registered", + ) + + if release.qualification_status in _WITHDRAWN: + return ReleaseResolution( + status=ReleaseResolutionStatus.WITHDRAWN, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message=f"release is {release.qualification_status.value}", + ) + + if release.qualification_status in _UNQUALIFIED: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message=( + "release is not qualified " + f"({release.qualification_status.value})" + ), + ) + + if release.is_byoa: + # A BYOA release is pinned by its command/package identity, not by an + # artifact digest; there is nothing to stage or verify up front. + if release.byoa_identity is None: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + distribution_mode=release.distribution_mode.value, + message=( + "qualified BYOA release has no command or package identity" + ), + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + distribution_mode=release.distribution_mode.value, + ) + + digest = self.artifact_digest(release) + if not digest: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message="qualified release has no resolvable artifact digest", + ) + + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + artifact_digest=digest, + distribution_mode=release.distribution_mode.value, + ) + + def artifact_digest(self, release: AgentReleaseV1) -> Optional[str]: + """Return the digest of the selected artifact, or ``None``. + + When a platform is configured it is selected exactly; otherwise the + release must publish exactly one artifact, or the first artifact in a + stable ``(os, arch)`` order is used. + """ + if self.platform is not None: + resolution = self.registry.resolve_artifact( + release, self.platform[0], self.platform[1] + ) + if not resolution.resolved or resolution.artifact is None: + return None + return resolution.artifact.sha256 + + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + if not artifacts: + return None + return artifacts[0].sha256 diff --git a/src/research/study/agents/store.py b/src/research/study/agents/store.py new file mode 100644 index 0000000..007edfa --- /dev/null +++ b/src/research/study/agents/store.py @@ -0,0 +1,205 @@ +"""CRUD-style persistence helpers for the agent registry. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. The router is +responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import AgentReleaseV1, CapabilitySnapshotV1 + +from database.research_schemas import AgentRelease, ResearchAgentRun + +from .registry import derive_qualification_status + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def upsert_release(session: Session, release: AgentReleaseV1) -> AgentRelease: + """Insert or update one release row. + + A release's artifacts and adapter are part of ``release_json`` (its + canonical content), so no child rows are written. Identity conflicts (same + release bytes) are left to the database's unique + ``(agent_id, source_manifest_digest)`` constraint; callers decide whether + that is a conflict or an idempotent retry. + + Qualification is **never** taken from the incoming model: it is derived from + the stored conformance evidence, so re-registering a release cannot promote + or downgrade it. + """ + row = session.get(AgentRelease, release.release_id) + release_json = release.model_dump(mode="json") + + if row is None: + merged = dict(release_json) + else: + merged = dict(row.release_json or {}) + # Packaging/conformance evidence shares release_json with the release + # model, so a re-registration must not drop those sibling keys. + merged.update(release_json) + row.agent_id = release.agent_id + row.source_manifest_digest = release.source_manifest_digest + + status = derive_qualification_status(merged) + merged["qualification_status"] = status.value + + if row is None: + row = AgentRelease( + release_id=release.release_id, + agent_id=release.agent_id, + source_manifest_digest=release.source_manifest_digest, + status=status.value, + release_json=merged, + created_at=release.created_at or _now(), + ) + session.add(row) + else: + row.status = status.value + row.release_json = merged + + session.commit() + session.refresh(row) + return row + + +def get_release(session: Session, release_id: str) -> Optional[AgentRelease]: + """Fetch a release row by primary key, or ``None``.""" + return session.get(AgentRelease, release_id) + + +def list_releases( + session: Session, agent_id: Optional[str] = None +) -> Sequence[AgentRelease]: + """List release rows (optionally for one agent) in a stable order.""" + statement = select(AgentRelease) + if agent_id is not None: + statement = statement.where(AgentRelease.agent_id == agent_id) + statement = statement.order_by( + AgentRelease.agent_id.asc(), AgentRelease.release_id.asc() + ) + return list(session.execute(statement).scalars().all()) + + +def insert_snapshot(session: Session, snapshot: CapabilitySnapshotV1) -> ResearchAgentRun: + """Store a capability snapshot on the run that captured it. + + A snapshot is a property of an agent run, so it is written to + ``research_agent_run.snapshot_json``. The admin qualification upload has no + participant session, so it records a detached run (``research_session_id`` + is ``None``) keyed by the snapshot id. + """ + payload = snapshot.model_dump(mode="json") + row = session.get(ResearchAgentRun, snapshot.snapshot_id) + if row is None: + row = ResearchAgentRun( + agent_run_id=snapshot.snapshot_id, + research_session_id=None, + agent_release_id=snapshot.release_id, + started_at=snapshot.captured_at, + snapshot_json=payload, + snapshot_captured_at=snapshot.captured_at, + ) + else: + row.snapshot_json = payload + row.snapshot_captured_at = snapshot.captured_at + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_snapshot( + session: Session, snapshot_id: uuid.UUID +) -> Optional[ResearchAgentRun]: + """Fetch the run holding a snapshot by snapshot id, or ``None``.""" + row = session.get(ResearchAgentRun, snapshot_id) + if row is None or row.snapshot_json is None: + return None + return row + + +def list_snapshots( + session: Session, release_id: Optional[str] = None +) -> Sequence[ResearchAgentRun]: + """List snapshot-bearing runs (optionally for one release), oldest-first.""" + statement = select(ResearchAgentRun).where( + ResearchAgentRun.snapshot_json.is_not(None) + ) + if release_id is not None: + statement = statement.where( + ResearchAgentRun.snapshot_json["release_id"].astext == release_id + ) + statement = statement.order_by(ResearchAgentRun.snapshot_captured_at.asc()) + return list(session.execute(statement).scalars().all()) + + +def row_to_release(row: AgentRelease) -> AgentReleaseV1: + """Rehydrate a stored release row into the versioned model. + + ``release_json`` also carries packaging/conformance evidence that is not + part of the release model, so only the model's own fields are validated; the + derived qualification status is then applied (never the stored raw value). + """ + from .models import AgentReleaseV1 as AgentReleaseV1Model + + payload = dict(row.release_json or {}) + model_fields = set(AgentReleaseV1Model.model_fields) + filtered = {key: value for key, value in payload.items() if key in model_fields} + release = AgentReleaseV1Model.model_validate(filtered) + return release.model_copy( + update={"qualification_status": derive_qualification_status(payload)} + ) + + +def row_to_snapshot(row: ResearchAgentRun) -> CapabilitySnapshotV1: + """Rehydrate a snapshot-bearing run row into the versioned model.""" + from .models import CapabilitySnapshotV1 as CapabilitySnapshotV1Model + + return CapabilitySnapshotV1Model.model_validate(row.snapshot_json) + + +def release_summary(row: AgentRelease) -> dict[str, Any]: + """Return a compact, non-secret release summary safe for list responses.""" + created_at = row.created_at + # The read path reports the derived status; the stored column is a cache of + # it, so recomputing from release_json keeps legacy rows truthful. + derived = derive_qualification_status(row.release_json) + return { + "release_id": row.release_id, + "agent_id": row.agent_id, + "source_manifest_digest": row.source_manifest_digest, + "status": derived.value, + "created_at": ( + created_at.isoformat() if isinstance(created_at, datetime) else None + ), + } + + +def snapshot_summary(row: ResearchAgentRun) -> dict[str, Any]: + """Return a compact snapshot summary safe for list responses.""" + payload = row.snapshot_json or {} + captured_at = row.snapshot_captured_at + return { + "snapshot_id": str(payload.get("snapshot_id") or row.agent_run_id), + "release_id": payload.get("release_id"), + "adapter_version": payload.get("adapter_version"), + "protocol_version": payload.get("protocol_version"), + "captured_at": ( + captured_at.isoformat() if isinstance(captured_at, datetime) else None + ), + } diff --git a/src/research/study/lifecycle.py b/src/research/study/lifecycle.py new file mode 100644 index 0000000..57864da --- /dev/null +++ b/src/research/study/lifecycle.py @@ -0,0 +1,111 @@ +"""Transactional lifecycle operations for research studies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional +import uuid + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from database.db_schemas import ResearchStudyStatus, Study +from database.research_schemas import ( + ResearchEnrollment, + ResearchSessionV1, + StudyAssignment, +) + + +@dataclass(frozen=True) +class StudyStopSummary: + """Counts and identity returned by a terminal study stop.""" + + study_id: uuid.UUID + enrollment_count: int + assignment_count: int + session_count: int + + +def stop_research_study( + session: Session, + study_id: uuid.UUID, + *, + actor: str, + now: Optional[datetime] = None, +) -> StudyStopSummary: + """Stop a research study without deleting retained research data.""" + study = session.execute( + select(Study).where(Study.study_id == study_id).with_for_update() + ).scalar_one_or_none() + if study is None or not bool(getattr(study, "is_research", False)): + raise ValueError("research study not found") + + timestamp = now or datetime.now(timezone.utc) + if getattr(study, "research_status", None) != ResearchStudyStatus.STUDY_STOPPED.value: + setattr(study, "research_status", ResearchStudyStatus.STUDY_STOPPED.value) + setattr(study, "is_active", False) + setattr(study, "stopped_at", timestamp) + setattr(study, "stopped_by", actor) + + enrollments = list( + session.execute( + select(ResearchEnrollment) + .where( + ResearchEnrollment.study_id == study_id, + ResearchEnrollment.status == "ACTIVE", + ) + .with_for_update() + ) + .scalars() + .all() + ) + enrollment_ids = [enrollment.enrollment_id for enrollment in enrollments] + for enrollment in enrollments: + setattr(enrollment, "status", "STUDY_STOPPED") + setattr( + enrollment, + "revocation_epoch", + int(getattr(enrollment, "revocation_epoch", 0)) + 1, + ) + setattr(enrollment, "updated_at", timestamp) + + assignments = [] + if enrollment_ids: + assignments = list( + session.execute( + select(StudyAssignment) + .where(StudyAssignment.enrollment_id.in_(enrollment_ids)) + .with_for_update() + ) + .scalars() + .all() + ) + for assignment in assignments: + setattr(assignment, "status", "STUDY_STOPPED") + + sessions = list( + session.execute( + select(ResearchSessionV1) + .where( + ResearchSessionV1.study_id == study_id, + ResearchSessionV1.state.not_in(("ended", "revoked")), + ) + .with_for_update() + ) + .scalars() + .all() + ) + for research_session in sessions: + setattr(research_session, "state", "revoked") + setattr(research_session, "closed_at", timestamp) + setattr(research_session, "close_reason", "STUDY_STOPPED") + + session.commit() + return StudyStopSummary( + study_id=study_id, + enrollment_count=len(enrollments), + assignment_count=len(assignments), + session_count=len(sessions), + ) diff --git a/src/research/study/packaging/__init__.py b/src/research/study/packaging/__init__.py new file mode 100644 index 0000000..2ead108 --- /dev/null +++ b/src/research/study/packaging/__init__.py @@ -0,0 +1,93 @@ +"""Runtime packaging and agent conformance suite (Issue 11). + +Public surface: + +* :mod:`research.study.packaging.enums` - conformance statuses and typed package + reason codes. +* :mod:`research.study.packaging.models` - ``RuntimeManifestV2``, ``ConformanceCaseV1``, + and ``ConformanceReceiptV1`` (digest-addressed). +* :mod:`research.study.packaging.verifier` - default-deny package verification and + path containment. +* :mod:`research.study.packaging.resolver` - exact platform selection with a required + bootstrap-digest match and no PATH/package-manager fallback. +* :mod:`research.study.packaging.conformance` - the capability-aware runner and the + release-qualification link. +* :mod:`research.study.packaging.packager` - synthetic package assembly helper. +* :mod:`research.study.packaging.store` - Session-supplied persistence helpers. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .conformance import ( + CaseObservation, + ConformanceObserver, + ConformanceRunner, + QualificationDecision, + qualification_for_release, +) +from .enums import ConformanceStatus, PackageReasonCode, PrerequisiteState +from .models import ( + ComponentEntry, + ConformanceCaseResultV1, + ConformanceCaseV1, + ConformanceReceiptV1, + PackageIssue, + PackageVerificationResult, + PlatformTriple, + ProtocolCompatibility, + ResolutionResult, + ResolvedComponent, + RuntimeManifestV2, + SelfCheckSpec, + manifest_digest_of, + normalize_sha256, + sha256_of_bytes, +) +from .packager import build_package, write_manifest +from .resolver import resolve_component, resolved_digest +from .verifier import ( + PathContainment, + hash_file, + resolve_under_root, + scan_secret_files, + scan_undeclared_executables, + sign_manifest, + verify_package, +) + +__all__ = [ + "CaseObservation", + "ComponentEntry", + "ConformanceCaseResultV1", + "ConformanceCaseV1", + "ConformanceObserver", + "ConformanceReceiptV1", + "ConformanceRunner", + "ConformanceStatus", + "PackageIssue", + "PackageReasonCode", + "PackageVerificationResult", + "PathContainment", + "PlatformTriple", + "PrerequisiteState", + "ProtocolCompatibility", + "QualificationDecision", + "ResolutionResult", + "ResolvedComponent", + "RuntimeManifestV2", + "SelfCheckSpec", + "build_package", + "hash_file", + "manifest_digest_of", + "normalize_sha256", + "qualification_for_release", + "resolve_component", + "resolve_under_root", + "resolved_digest", + "scan_secret_files", + "scan_undeclared_executables", + "sha256_of_bytes", + "sign_manifest", + "verify_package", + "write_manifest", +] diff --git a/src/research/study/packaging/conformance.py b/src/research/study/packaging/conformance.py new file mode 100644 index 0000000..7637817 --- /dev/null +++ b/src/research/study/packaging/conformance.py @@ -0,0 +1,347 @@ +"""Capability-aware conformance runner and qualification linking (Issue 11). + +The runner executes only cases whose prerequisites are established by injected +receipt/fixture evidence. A missing capability yields ``UNSUPPORTED``/``UNKNOWN``, +never ``PASS``. A case that mutates state without a successful cleanup yields +``FAIL``. Results are bound to the exact artifact/adapter/host/plugin/protocol/ +fixture digests by :class:`ConformanceReceiptV1`. + +The qualification link promotes an agent release only when a receipt for the +claimed platform/host proves every required case ``PASS``; it never auto-promotes. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Mapping, Optional, Protocol, Sequence +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash + +from .enums import ConformanceStatus, PackageReasonCode, PrerequisiteState +from .models import ( + ConformanceCaseResultV1, + ConformanceCaseV1, + ConformanceReceiptV1, + PlatformTriple, + normalize_sha256, +) + +if TYPE_CHECKING: + from research.study.agents.models import AgentReleaseV1 + +__all__ = [ + "CaseObservation", + "ConformanceObserver", + "ConformanceRunner", + "QualificationDecision", + "qualification_for_release", +] + +_BASE = ConfigDict(extra="forbid") + +#: Worst-first precedence for a receipt's aggregate status. +_STATUS_PRECEDENCE = ( + ConformanceStatus.FAIL, + ConformanceStatus.BLOCKED, + ConformanceStatus.UNKNOWN, + ConformanceStatus.UNSUPPORTED, + ConformanceStatus.PASS, +) + + +class CaseObservation(BaseModel): + """What an observer observed while executing one case.""" + + model_config = _BASE + + host_observations: list[str] = Field(default_factory=list) + agent_observations: list[str] = Field(default_factory=list) + cleanup_ok: Optional[bool] = None + performance_ms: Optional[float] = None + evidence: dict[str, str] = Field(default_factory=dict) + status: Optional[ConformanceStatus] = None + error: Optional[str] = None + + +class ConformanceObserver(Protocol): + """Executes one conformance case and returns its observations.""" + + def observe(self, case: ConformanceCaseV1) -> CaseObservation: # pragma: no cover - protocol + ... + + +def _coerce_state(value: object) -> PrerequisiteState: + if isinstance(value, PrerequisiteState): + return value + text = str(getattr(value, "value", value)).strip().upper() + try: + return PrerequisiteState(text) + except ValueError: + return PrerequisiteState.UNKNOWN + + +class ConformanceRunner: + """Runs conformance cases with explicit, truthful statuses.""" + + def __init__(self, observer: ConformanceObserver) -> None: + self._observer = observer + + def run( + self, + cases: Sequence[ConformanceCaseV1], + prerequisites: Mapping[str, object], + *, + artifact_digest: str, + adapter_digest: str, + host: PlatformTriple, + plugin_version: str, + protocol_version: str, + fixture_digests: Optional[Mapping[str, str]] = None, + now: Optional[datetime] = None, + receipt_id: Optional[UUID] = None, + ) -> ConformanceReceiptV1: + """Execute prerequisite-established cases and return a bound receipt.""" + states = {name: _coerce_state(value) for name, value in prerequisites.items()} + results = [self._run_case(case, states) for case in cases] + status = self._aggregate(results) + return ConformanceReceiptV1( + receipt_id=receipt_id or uuid.uuid4(), + artifact_digest=artifact_digest, + adapter_digest=adapter_digest, + host=host, + plugin_version=plugin_version, + protocol_version=protocol_version, + fixture_digests=dict(fixture_digests or {}), + case_results=results, + status=status, + created_at=now or datetime.now(timezone.utc), + ) + + def _run_case( + self, case: ConformanceCaseV1, states: Mapping[str, PrerequisiteState] + ) -> ConformanceCaseResultV1: + missing = [name for name in case.prerequisites if states.get(name) != PrerequisiteState.ESTABLISHED] + if missing: + unsupported = [ + name + for name in missing + if states.get(name) == PrerequisiteState.UNSUPPORTED + ] + if unsupported: + return self._result( + case, + ConformanceStatus.UNSUPPORTED, + reason="capability is not supported on this host/agent: " + + ", ".join(sorted(unsupported)), + ) + return self._result( + case, + ConformanceStatus.UNKNOWN, + reason="capability is not established: " + ", ".join(sorted(missing)), + ) + + try: + observation = self._observer.observe(case) + except Exception as error: # noqa: BLE001 - an observer failure blocks the case + return self._result( + case, ConformanceStatus.BLOCKED, reason=f"observer raised: {error}" + ) + + if observation.error: + return self._result( + case, ConformanceStatus.BLOCKED, observation, reason=observation.error + ) + + # A mutation without a verified cleanup is a failure, never a pass. + if observation.cleanup_ok is False: + return self._result( + case, + ConformanceStatus.FAIL, + observation, + reason="case mutated state without a successful cleanup", + ) + + if observation.status is not None and observation.status != ConformanceStatus.PASS: + return self._result( + case, + observation.status, + observation, + reason="observer reported a non-pass status", + ) + + if case.cleanup_assertion and observation.cleanup_ok is None: + return self._result( + case, + ConformanceStatus.UNKNOWN, + observation, + reason="cleanup assertion was not verified", + ) + + missing_host = [ + expected + for expected in case.expected_host_observations + if expected not in observation.host_observations + ] + missing_agent = [ + expected + for expected in case.expected_agent_observations + if expected not in observation.agent_observations + ] + if missing_host or missing_agent: + return self._result( + case, + ConformanceStatus.FAIL, + observation, + reason="expected observations were not all present", + ) + + if ( + case.max_performance_ms is not None + and observation.performance_ms is not None + and observation.performance_ms > case.max_performance_ms + ): + return self._result( + case, + ConformanceStatus.FAIL, + observation, + reason=( + f"performance {observation.performance_ms}ms exceeded bound " + f"{case.max_performance_ms}ms" + ), + ) + + return self._result(case, ConformanceStatus.PASS, observation) + + @staticmethod + def _result( + case: ConformanceCaseV1, + status: ConformanceStatus, + observation: Optional[CaseObservation] = None, + *, + reason: str = "", + ) -> ConformanceCaseResultV1: + observed = observation or CaseObservation() + evidence_digest = canonical_hash( + { + "case_id": case.case_id, + "status": status.value, + "host_observations": observed.host_observations, + "agent_observations": observed.agent_observations, + "cleanup_ok": observed.cleanup_ok, + "evidence": observed.evidence, + "reason": reason, + } + ) + return ConformanceCaseResultV1( + case_id=case.case_id, + status=status, + host_observations=list(observed.host_observations), + agent_observations=list(observed.agent_observations), + cleanup_ok=observed.cleanup_ok, + performance_ms=observed.performance_ms, + evidence_digest=evidence_digest, + reason=reason, + ) + + @staticmethod + def _aggregate(results: Sequence[ConformanceCaseResultV1]) -> ConformanceStatus: + if not results: + return ConformanceStatus.UNKNOWN + seen = {result.status for result in results} + for status in _STATUS_PRECEDENCE: + if status in seen: + return status + return ConformanceStatus.UNKNOWN + + +class QualificationDecision(BaseModel): + """Typed outcome of linking conformance evidence to release qualification.""" + + model_config = _BASE + + promote: bool + reason: PackageReasonCode + receipt_id: Optional[UUID] = None + message: str = "" + + +def qualification_for_release( + release: AgentReleaseV1, + receipts: Sequence[ConformanceReceiptV1], + *, + required_cases: Sequence[str], + os: str, + arch: str, +) -> QualificationDecision: + """Decide whether a release may be promoted for one host/platform tuple. + + Promotion requires a receipt that binds the exact release artifact digest and + adapter digest for ``(os, arch)`` and proves every ``required_cases`` entry + ``PASS``. Anything else is an explicit non-promotion. + """ + artifact = next( + (item for item in release.artifacts if item.os == os and item.arch == arch), + None, + ) + if artifact is None: + return QualificationDecision( + promote=False, + reason=PackageReasonCode.MISSING_COMPONENT, + message=f"release has no artifact for '{os}-{arch}'", + ) + if release.adapter is None: + return QualificationDecision( + promote=False, + reason=PackageReasonCode.CONFORMANCE_NOT_PASSED, + message="release declares no adapter identity", + ) + + artifact_hex = normalize_sha256(artifact.sha256) + adapter_hex = normalize_sha256(release.adapter.digest) + required = set(required_cases) + + candidates = [ + receipt + for receipt in receipts + if normalize_sha256(receipt.artifact_digest) == artifact_hex + and receipt.adapter_digest == release.adapter.digest + ] + if not candidates: + return QualificationDecision( + promote=False, + reason=PackageReasonCode.CONFORMANCE_NOT_PASSED, + message="no conformance receipt is bound to this artifact/adapter", + ) + + host_matched = [ + receipt + for receipt in candidates + if receipt.host.os == os and receipt.host.arch == arch + ] + if not host_matched: + return QualificationDecision( + promote=False, + reason=PackageReasonCode.RECEIPT_HOST_MISMATCH, + message=f"no conformance receipt was recorded for '{os}-{arch}'", + ) + + for receipt in host_matched: + if adapter_hex is not None and normalize_sha256(receipt.adapter_digest) != adapter_hex: + continue + if required.issubset(receipt.passed_cases()): + return QualificationDecision( + promote=True, + reason=PackageReasonCode.OK, + receipt_id=receipt.receipt_id, + message="all required cases passed for the claimed platform/host", + ) + + return QualificationDecision( + promote=False, + reason=PackageReasonCode.CONFORMANCE_NOT_PASSED, + message="required conformance cases did not all pass for this platform/host", + ) diff --git a/src/research/study/packaging/enums.py b/src/research/study/packaging/enums.py new file mode 100644 index 0000000..c2f4877 --- /dev/null +++ b/src/research/study/packaging/enums.py @@ -0,0 +1,77 @@ +"""Closed vocabularies for runtime packaging and conformance (Issue 11). + +Every member is part of a persisted or reported contract (package manifests, +verification results, conformance receipts). Members are additive only, and the +conformance statuses preserve ``UNSUPPORTED``/``UNKNOWN`` so a missing capability +can never be mistaken for a pass. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["ConformanceStatus", "PackageReasonCode", "PrerequisiteState"] + + +class ConformanceStatus(str, Enum): + """Truthful outcome of one conformance case (or a whole receipt). + + ``PASS`` is the only status that supports a qualified advertised behavior. + ``UNSUPPORTED``/``UNKNOWN`` are explicit non-passes, and ``BLOCKED`` means + the case could not be exercised at all. + """ + + PASS = "PASS" + FAIL = "FAIL" + UNSUPPORTED = "UNSUPPORTED" + UNKNOWN = "UNKNOWN" + BLOCKED = "BLOCKED" + + +class PrerequisiteState(str, Enum): + """Whether a case prerequisite is established, unsupported, or unknown.""" + + ESTABLISHED = "ESTABLISHED" + UNSUPPORTED = "UNSUPPORTED" + UNKNOWN = "UNKNOWN" + + +class PackageReasonCode(str, Enum): + """Stable machine-readable reason a package/component was blocked.""" + + OK = "OK" + + # Manifest structure. + INVALID_MANIFEST = "INVALID_MANIFEST" + SCHEMA_VERSION_UNSUPPORTED = "SCHEMA_VERSION_UNSUPPORTED" + EMPTY_COMPONENTS = "EMPTY_COMPONENTS" + DUPLICATE_PLATFORM = "DUPLICATE_PLATFORM" + + # Path containment. + PATH_ESCAPE = "PATH_ESCAPE" + PATH_UNSAFE = "PATH_UNSAFE" + + # Integrity. + DIGEST_MISMATCH = "DIGEST_MISMATCH" + SIZE_MISMATCH = "SIZE_MISMATCH" + SIGNATURE_MISMATCH = "SIGNATURE_MISMATCH" + ARTIFACT_MISSING = "ARTIFACT_MISSING" + MISSING_COMPONENT = "MISSING_COMPONENT" + NOT_EXECUTABLE = "NOT_EXECUTABLE" + + # Archive hygiene. + UNDECLARED_EXECUTABLE = "UNDECLARED_EXECUTABLE" + SECRET_FILE_PRESENT = "SECRET_FILE_PRESENT" + + # Selection. + UNSUPPORTED_PLATFORM = "UNSUPPORTED_PLATFORM" + BOOTSTRAP_DIGEST_MISMATCH = "BOOTSTRAP_DIGEST_MISMATCH" + + # Self-check / tooling. + SELF_CHECK_FAILED = "SELF_CHECK_FAILED" + WRITER_UNAVAILABLE = "WRITER_UNAVAILABLE" + + # Qualification. + CONFORMANCE_NOT_PASSED = "CONFORMANCE_NOT_PASSED" + RECEIPT_HOST_MISMATCH = "RECEIPT_HOST_MISMATCH" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" diff --git a/src/research/study/packaging/models.py b/src/research/study/packaging/models.py new file mode 100644 index 0000000..8b1fd74 --- /dev/null +++ b/src/research/study/packaging/models.py @@ -0,0 +1,352 @@ +"""Pydantic v2 contracts for runtime packaging and conformance (Issue 11). + +``RuntimeManifestV2`` is authoritative for the exact contents and integrity of a +participant runtime package. ``ConformanceCaseV1``/``ConformanceReceiptV1`` bind +observed results to the exact artifact/adapter/host/plugin/protocol/fixture +digests, and preserve ``UNSUPPORTED``/``UNKNOWN`` rather than upgrading them. + +Digest convention: component payload digests and the manifest digest are +``sha256:<64 lowercase hex>`` so they compare directly with the agent +``DistributionArtifact.sha256`` and the bootstrap ``artifact_digest``. A bare hex +digest is also accepted when comparing (see :func:`normalize_sha256`). +""" + +from __future__ import annotations + +import hashlib +from datetime import datetime # noqa: TC003 - pydantic resolves annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.study.agents.models import AdapterRef + +from .enums import ( + ConformanceStatus, + PackageReasonCode, # noqa: TC001 - pydantic resolves annotations at runtime +) + +_BASE = ConfigDict(extra="forbid") + +__all__ = [ + "AdapterRef", + "ComponentEntry", + "ConformanceCaseV1", + "ConformanceCaseResultV1", + "ConformanceReceiptV1", + "PackageIssue", + "PackageVerificationResult", + "PlatformTriple", + "ProtocolCompatibility", + "ResolutionResult", + "ResolvedComponent", + "RuntimeManifestV2", + "SelfCheckSpec", + "manifest_digest_of", + "normalize_sha256", + "sha256_of_bytes", +] + + +def sha256_of_bytes(data: bytes) -> str: + """Return ``sha256:`` for ``data``.""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def normalize_sha256(value: Optional[str]) -> Optional[str]: + """Return the bare lowercase hex of ``value`` (stripping ``sha256:``), or ``None``.""" + if value is None: + return None + text = value.strip().lower() + if text.startswith("sha256:"): + text = text[len("sha256:") :] + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + return None + return text + + +class PlatformTriple(BaseModel): + """An ``(os, arch)`` selection key.""" + + model_config = _BASE + + os: str + arch: str + + @property + def key(self) -> str: + """The ``os-arch`` tuple used in diagnostics and args templates.""" + return f"{self.os}-{self.arch}" + + +class ComponentEntry(BaseModel): + """One platform-specific component payload inside a package.""" + + model_config = _BASE + + name: str + os: str + arch: str + relative_path: str + sha256: str + size: int + executable: bool = False + license: Optional[str] = None + + @property + def platform(self) -> PlatformTriple: + """The component's platform triple.""" + return PlatformTriple(os=self.os, arch=self.arch) + + +class SelfCheckSpec(BaseModel): + """A declared self-check command (never a shell string).""" + + model_config = _BASE + + command: list[str] = Field(default_factory=list) + expected_exit_code: int = 0 + + +class ProtocolCompatibility(BaseModel): + """ACP protocol compatibility range for the packaged runtime.""" + + model_config = _BASE + + min_protocol_version: Optional[str] = None + max_protocol_version: Optional[str] = None + + def supports(self, protocol_version: Optional[str]) -> Optional[bool]: + """Whether ``protocol_version`` is in range; ``None`` when undecidable.""" + if protocol_version is None: + return None + if self.min_protocol_version is None and self.max_protocol_version is None: + return None + current = _version_tuple(protocol_version) + if current is None: + return None + if self.min_protocol_version is not None: + minimum = _version_tuple(self.min_protocol_version) + if minimum is not None and current < minimum: + return False + if self.max_protocol_version is not None: + maximum = _version_tuple(self.max_protocol_version) + if maximum is not None and current > maximum: + return False + return True + + +def _version_tuple(value: str) -> Optional[tuple[int, ...]]: + try: + parts = [int(part) for part in value.strip().split(".")] + except (TypeError, ValueError): + return None + return tuple(parts) if parts else None + + +class RuntimeManifestV2(BaseModel): + """The package manifest: authoritative for contents and integrity.""" + + model_config = _BASE + + schema_version: str = "2" + manifest_digest: str = "" + release_id: str + agent_id: str + adapter_ref: AdapterRef + components: list[ComponentEntry] = Field(default_factory=list) + supported_platforms: list[PlatformTriple] = Field(default_factory=list) + # Argument template keyed by platform key ("macos-arm64"); always an array. + args_template: dict[str, list[str]] = Field(default_factory=dict) + licenses: list[str] = Field(default_factory=list) + compatibility: ProtocolCompatibility = Field(default_factory=ProtocolCompatibility) + self_check: Optional[SelfCheckSpec] = None + signature: Optional[str] = None + + def platform_keys(self) -> list[str]: + """Supported platform keys in declared order.""" + return [platform.key for platform in self.supported_platforms] + + def supports_platform(self, os: str, arch: str) -> bool: + """Whether ``(os, arch)`` is a declared supported platform.""" + return any( + platform.os == os and platform.arch == arch + for platform in self.supported_platforms + ) + + def component_for(self, os: str, arch: str) -> Optional[ComponentEntry]: + """The first component whose ``(os, arch)`` exactly matches, or ``None``.""" + return next( + ( + component + for component in self.components + if component.os == os and component.arch == arch + ), + None, + ) + + def executable_component_for(self, os: str, arch: str) -> Optional[ComponentEntry]: + """The runnable component for ``(os, arch)``, or the first, or ``None``.""" + matches = [ + component + for component in self.components + if component.os == os and component.arch == arch + ] + return next((c for c in matches if c.executable), matches[0] if matches else None) + + def args_for_platform(self, os: str, arch: str) -> list[str]: + """Argument-array template for a platform (falls back to ``os`` then empty).""" + key = f"{os}-{arch}" + if key in self.args_template: + return list(self.args_template[key]) + if os in self.args_template: + return list(self.args_template[os]) + return [] + + def canonical_payload(self) -> dict: + """Manifest fields except ``manifest_digest`` and ``signature``. + + Both the digest and the HMAC are computed over this payload, so a signed + manifest can also be digest-verified without a circular dependency. + """ + payload = self.model_dump(mode="json") + payload.pop("manifest_digest", None) + payload.pop("signature", None) + return payload + + def computed_digest(self) -> str: + """``sha256:`` over the canonical payload.""" + return "sha256:" + canonical_hash(self.canonical_payload()) + + def digest_matches(self) -> bool: + """Whether the stored digest matches the recomputed payload digest.""" + stored = normalize_sha256(self.manifest_digest) + return stored is not None and stored == normalize_sha256(self.computed_digest()) + + def with_digest(self) -> "RuntimeManifestV2": + """Return a copy with ``manifest_digest`` set to the computed value.""" + return self.model_copy(update={"manifest_digest": self.computed_digest()}) + + def platform_conflicts(self) -> list[str]: + """Platform keys declared by more than one component.""" + seen: dict[str, int] = {} + for component in self.components: + seen[component.platform.key] = seen.get(component.platform.key, 0) + 1 + return sorted(key for key, count in seen.items() if count > 1) + + def supports_protocol(self, protocol_version: Optional[str]) -> Optional[bool]: + """Whether the packaged runtime supports ``protocol_version``.""" + return self.compatibility.supports(protocol_version) + + +def manifest_digest_of(payload: dict) -> str: + """Compute the manifest digest for a raw payload dict (``manifest_digest``/``signature`` removed).""" + trimmed = dict(payload) + trimmed.pop("manifest_digest", None) + trimmed.pop("signature", None) + return "sha256:" + canonical_hash(trimmed) + + +class PackageIssue(BaseModel): + """One typed package/verification finding.""" + + model_config = _BASE + + code: PackageReasonCode + message: str + field: str = "" + + +class PackageVerificationResult(BaseModel): + """Typed aggregate result of verifying a package directory.""" + + model_config = _BASE + + valid: bool + manifest_digest: Optional[str] = None + verified_components: int = 0 + errors: list[PackageIssue] = Field(default_factory=list) + + @property + def codes(self) -> set[PackageReasonCode]: + """The set of reason codes observed.""" + return {error.code for error in self.errors} + + +class ResolvedComponent(BaseModel): + """A component selected for one exact platform and bootstrap digest.""" + + model_config = _BASE + + component: ComponentEntry + os: str + arch: str + arguments: list[str] = Field(default_factory=list) + relative_path: str + + +class ResolutionResult(BaseModel): + """Typed result of resolving a component with no fallback.""" + + model_config = _BASE + + resolved: Optional[ResolvedComponent] = None + error: Optional[PackageIssue] = None + + +class ConformanceCaseV1(BaseModel): + """One capability-aware conformance case.""" + + model_config = _BASE + + case_id: str + prerequisites: list[str] = Field(default_factory=list) + fixture_ref: str = "" + action_steps: list[str] = Field(default_factory=list) + expected_host_observations: list[str] = Field(default_factory=list) + expected_agent_observations: list[str] = Field(default_factory=list) + cleanup_assertion: str = "" + max_performance_ms: Optional[float] = None + status: ConformanceStatus = ConformanceStatus.UNKNOWN + + +class ConformanceCaseResultV1(BaseModel): + """The observed result of one conformance case.""" + + model_config = _BASE + + case_id: str + status: ConformanceStatus + host_observations: list[str] = Field(default_factory=list) + agent_observations: list[str] = Field(default_factory=list) + cleanup_ok: Optional[bool] = None + performance_ms: Optional[float] = None + evidence_digest: str = "" + reason: str = "" + + +class ConformanceReceiptV1(BaseModel): + """Binds conformance results to exact artifact/adapter/host/plugin/protocol/fixtures.""" + + model_config = _BASE + + receipt_id: UUID + artifact_digest: str + adapter_digest: str + host: PlatformTriple + plugin_version: str + protocol_version: str + fixture_digests: dict[str, str] = Field(default_factory=dict) + case_results: list[ConformanceCaseResultV1] = Field(default_factory=list) + status: ConformanceStatus = ConformanceStatus.UNKNOWN + created_at: datetime + + def passed_cases(self) -> set[str]: + """Case ids whose observed status is exactly ``PASS``.""" + return { + result.case_id + for result in self.case_results + if result.status == ConformanceStatus.PASS + } diff --git a/src/research/study/packaging/packager.py b/src/research/study/packaging/packager.py new file mode 100644 index 0000000..a91fc76 --- /dev/null +++ b/src/research/study/packaging/packager.py @@ -0,0 +1,97 @@ +"""Synthetic package assembly helper (Issue 11). + +``build_package`` writes a directory of files and returns a fully-populated, +digest-correct :class:`RuntimeManifestV2`. It exists so tests (and a future CI +builder) can assemble a package without any network or host state. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Mapping, Optional, Sequence + +from .models import ( + ComponentEntry, + PlatformTriple, + ProtocolCompatibility, + RuntimeManifestV2, + SelfCheckSpec, + sha256_of_bytes, +) +from .verifier import sign_manifest + +if TYPE_CHECKING: + from pathlib import Path + + from research.study.agents.models import AdapterRef + +__all__ = ["build_package", "write_manifest"] + + +def build_package( + root: Path, + *, + release_id: str, + agent_id: str, + adapter_ref: AdapterRef, + os: str, + arch: str, + files: Mapping[str, bytes], + executable: Optional[str] = None, + component_name: str = "runtime", + args_template: Optional[Sequence[str]] = None, + licenses: Optional[Sequence[str]] = None, + compatibility: Optional[ProtocolCompatibility] = None, + self_check: Optional[SelfCheckSpec] = None, + sign_secret: Optional[str] = None, +) -> RuntimeManifestV2: + """Write ``files`` under ``root`` and return a digest-correct manifest.""" + root.mkdir(parents=True, exist_ok=True) + for relative, content in files.items(): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + if relative == executable: + target.chmod(target.stat().st_mode | 0o111) + + if executable is not None and executable not in files: + raise ValueError(f"executable '{executable}' is not one of the package files") + + # One component per payload file keeps every declared path exact. + components = [ + ComponentEntry( + name=component_name if relative == (executable or "") else f"{component_name}:{relative}", + os=os, + arch=arch, + relative_path=relative, + sha256=sha256_of_bytes(content), + size=len(content), + executable=(relative == executable), + ) + for relative, content in files.items() + ] + + manifest = RuntimeManifestV2( + schema_version="2", + release_id=release_id, + agent_id=agent_id, + adapter_ref=adapter_ref, + components=components, + supported_platforms=[PlatformTriple(os=os, arch=arch)], + args_template={f"{os}-{arch}": list(args_template or [])}, + licenses=list(licenses or []), + compatibility=compatibility or ProtocolCompatibility(), + self_check=self_check, + ) + if sign_secret: + # Sign the unsigned payload, then digest the signed manifest. + manifest = manifest.model_copy(update={"signature": sign_manifest(manifest, sign_secret)}) + return manifest.with_digest() + + +def write_manifest(path: Path, manifest: RuntimeManifestV2) -> Path: + """Write a manifest as canonical JSON to ``path``.""" + from research.canonical import canonical_json + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(canonical_json(manifest.model_dump(mode="json")), encoding="utf-8") + return path diff --git a/src/research/study/packaging/resolver.py b/src/research/study/packaging/resolver.py new file mode 100644 index 0000000..ba5ac77 --- /dev/null +++ b/src/research/study/packaging/resolver.py @@ -0,0 +1,95 @@ +"""Exact runtime component selection (Issue 11). + +``resolve_component`` selects the component for one exact ``(os, arch)`` and +requires its digest to equal the digest the bootstrap manifest declared. There is +deliberately **no fallback**: no ``PATH`` lookup, no ``npm``/``npx``, no source +checkout, no other platform or version. A failure returns a typed block and no +component, so a caller can never launch an unaudited executable. +""" + +from __future__ import annotations + +from typing import Optional + +from .enums import PackageReasonCode +from .models import ( + PackageIssue, + ResolutionResult, + ResolvedComponent, + RuntimeManifestV2, + normalize_sha256, +) + +__all__ = ["resolve_component", "resolved_digest"] + + +def resolve_component( + manifest: RuntimeManifestV2, + os: str, + arch: str, + bootstrap_artifact_digest: str, +) -> ResolutionResult: + """Resolve the exact component, or return a typed block (never a fallback).""" + if not os or not arch: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.UNSUPPORTED_PLATFORM, + message="a non-blank os and arch are required to resolve a runtime", + ) + ) + if not manifest.supports_platform(os, arch): + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.UNSUPPORTED_PLATFORM, + message=f"manifest does not declare platform '{os}-{arch}'", + field="supported_platforms", + ) + ) + component = manifest.executable_component_for(os, arch) + if component is None: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.MISSING_COMPONENT, + message=f"manifest has no component for '{os}-{arch}'", + field="components", + ) + ) + + declared = normalize_sha256(component.sha256) + if declared is None: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.DIGEST_MISMATCH, + message=f"component '{component.name}' has a malformed sha256", + field="components.sha256", + ) + ) + expected = normalize_sha256(bootstrap_artifact_digest) + if expected is None or expected != declared: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.BOOTSTRAP_DIGEST_MISMATCH, + message=( + f"component '{component.name}' digest does not match the " + "bootstrap-declared artifact digest" + ), + field="artifact_digest", + ) + ) + + return ResolutionResult( + resolved=ResolvedComponent( + component=component, + os=os, + arch=arch, + arguments=manifest.args_for_platform(os, arch), + relative_path=component.relative_path, + ) + ) + + +def resolved_digest(result: ResolutionResult) -> Optional[str]: + """Convenience: the resolved component digest, or ``None`` when blocked.""" + if result.resolved is None: + return None + return normalize_sha256(result.resolved.component.sha256) diff --git a/src/research/study/packaging/store.py b/src/research/study/packaging/store.py new file mode 100644 index 0000000..3da57dd --- /dev/null +++ b/src/research/study/packaging/store.py @@ -0,0 +1,339 @@ +"""Persistence helpers for runtime packages and conformance receipts (Issue 11). + +Packaging evidence is not a table: a release's ``RuntimeManifestV2`` lives in +``agent_release.release_json.package_json`` and its immutable conformance +receipts live in ``agent_release.release_json.conformance[]``. Every helper +takes a caller-supplied SQLAlchemy ``Session`` and works in domain models +(:class:`RuntimeManifestV2`, :class:`ConformanceReceiptV1`), never raw ORM rows. +The packaging package itself imports no FastAPI/App. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +from database.research_schemas import AgentRelease +from research.study.agents.registry import derive_qualification_status + +from .models import ConformanceReceiptV1, RuntimeManifestV2, normalize_sha256 + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +__all__ = [ + "PackageView", + "conformance_coverage", + "get_package", + "get_package_by_digest", + "get_receipt", + "insert_receipt", + "list_packages", + "list_receipts", + "package_summary", + "row_to_manifest", + "row_to_receipt", + "upsert_package", +] + +#: ``release_json`` keys that carry packaging evidence. +PACKAGE_KEY = "package_json" +CONFORMANCE_KEY = "conformance" + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +@dataclass(frozen=True) +class PackageView: + """One release's stored runtime package (a projection of ``release_json``).""" + + package_id: uuid.UUID + release_id: str + os: Optional[str] + arch: Optional[str] + manifest_digest: str + manifest_json: dict[str, Any] + signature: Optional[str] + created_at: Optional[datetime] + + +def _package_id(manifest_digest: str) -> uuid.UUID: + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://runtime-package/{manifest_digest}" + ) + + +def _package_from_release(row: AgentRelease) -> Optional[PackageView]: + data = row.release_json or {} + manifest = data.get(PACKAGE_KEY) + if not isinstance(manifest, dict): + return None + platforms = manifest.get("supported_platforms") or [] + platform = platforms[0] if platforms else {} + manifest_digest = str(manifest.get("manifest_digest") or "") + return PackageView( + package_id=_package_id(manifest_digest), + release_id=row.release_id, + os=platform.get("os"), + arch=platform.get("arch"), + manifest_digest=manifest_digest, + manifest_json=manifest, + signature=manifest.get("signature"), + created_at=row.created_at, + ) + + +def _iter_packages(session: Session) -> list[PackageView]: + statement = select(AgentRelease) + views = [ + _package_from_release(row) + for row in session.execute(statement).scalars().all() + ] + return [view for view in views if view is not None] + + +def upsert_package( + session: Session, + manifest: RuntimeManifestV2, + *, + package_id: Optional[uuid.UUID] = None, +) -> PackageView: + """Store a release's package manifest, or return the existing digest view. + + The package is folded into ``agent_release.release_json.package_json`` (the + release is created in ``DRAFT`` if it is not registered yet). ``package_id`` + is derived from the manifest digest, so it is deterministic. + """ + existing = get_package_by_digest(session, manifest.manifest_digest) + if existing is not None: + return existing + row = session.get(AgentRelease, manifest.release_id) + if row is None: + row = AgentRelease( + release_id=manifest.release_id, + agent_id=manifest.agent_id, + source_manifest_digest=manifest.manifest_digest, + status="UNQUALIFIED", + release_json={}, + created_at=_now(), + ) + session.add(row) + data = dict(row.release_json or {}) + data[PACKAGE_KEY] = manifest.model_dump(mode="json") + derived = derive_qualification_status(data) + data["qualification_status"] = derived.value + row.status = derived.value + row.release_json = data + session.commit() + session.refresh(row) + view = _package_from_release(row) + assert view is not None + return view + + +def get_package(session: Session, package_id: uuid.UUID) -> Optional[PackageView]: + """Fetch a package view by its deterministic package id, or ``None``.""" + for view in _iter_packages(session): + if view.package_id == package_id: + return view + return None + + +def get_package_by_digest( + session: Session, manifest_digest: str +) -> Optional[PackageView]: + """Fetch a package view by its manifest digest, or ``None``.""" + for view in _iter_packages(session): + if view.manifest_digest == manifest_digest: + return view + return None + + +def list_packages( + session: Session, release_id: Optional[str] = None +) -> Sequence[PackageView]: + """List packages, optionally for one release, oldest-first.""" + views = [ + view + for view in _iter_packages(session) + if release_id is None or view.release_id == release_id + ] + return sorted( + views, + key=lambda view: (view.created_at or datetime.min.replace(tzinfo=timezone.utc)), + ) + + +def row_to_manifest(view: PackageView) -> RuntimeManifestV2: + """Rehydrate a package view into its manifest.""" + return RuntimeManifestV2.model_validate(view.manifest_json) + + +def package_summary(view: PackageView) -> dict[str, Any]: + """Compact, non-secret package summary safe for responses.""" + manifest = view.manifest_json or {} + return { + "package_id": str(view.package_id), + "release_id": view.release_id, + "os": view.os, + "arch": view.arch, + "manifest_digest": view.manifest_digest, + "signature": view.signature, + "component_count": len(manifest.get("components") or []), + "supported_platforms": manifest.get("supported_platforms") or [], + "created_at": view.created_at.isoformat() if view.created_at else None, + } + + +def _same_digest(left: Optional[str], right: Optional[str]) -> bool: + return normalize_sha256(left) is not None and normalize_sha256( + left + ) == normalize_sha256(right) + + +def _find_release_for_artifact( + session: Session, artifact_digest: str +) -> Optional[AgentRelease]: + """Find the release that owns an exact artifact/adapter digest.""" + statement = select(AgentRelease) + for row in session.execute(statement).scalars().all(): + data = row.release_json or {} + for artifact in data.get("artifacts") or []: + if _same_digest(artifact.get("sha256"), artifact_digest): + return row + manifest = data.get(PACKAGE_KEY) or {} + for component in manifest.get("components") or []: + if _same_digest(component.get("sha256"), artifact_digest): + return row + # A BYOA release has no distribution artifact, so its conformance + # evidence binds to the release's own distribution digest instead. This + # keeps evidence-driven qualification working for participant-installed + # agents without inventing a fake artifact. + if not (data.get("artifacts") or []) and _same_digest( + data.get("source_manifest_digest"), artifact_digest + ): + return row + return None + + +def insert_receipt( + session: Session, receipt: ConformanceReceiptV1 +) -> ConformanceReceiptV1: + """Append one immutable conformance receipt to its release's ``release_json``. + + The release is located by the exact artifact digest the receipt is bound to. + """ + row = _find_release_for_artifact(session, receipt.artifact_digest) + if row is None: + raise LookupError( + "no registered release owns artifact digest " + f"{receipt.artifact_digest!r}" + ) + data = dict(row.release_json or {}) + conformance = [ + item + for item in (data.get(CONFORMANCE_KEY) or []) + if str(item.get("receipt_id")) != str(receipt.receipt_id) + ] + conformance.append(receipt.model_dump(mode="json")) + data[CONFORMANCE_KEY] = conformance + # Qualification is derived: recording a passing receipt is the only way a + # release becomes QUALIFIED, and the derived value is persisted so the read + # path never has to trust a caller-supplied status. + derived = derive_qualification_status(data) + data["qualification_status"] = derived.value + row.status = derived.value + row.release_json = data + session.add(row) + session.commit() + session.refresh(row) + return receipt + + +def _iter_receipts(session: Session) -> list[ConformanceReceiptV1]: + statement = select(AgentRelease) + receipts: list[ConformanceReceiptV1] = [] + for row in session.execute(statement).scalars().all(): + for item in (row.release_json or {}).get(CONFORMANCE_KEY) or []: + receipts.append(ConformanceReceiptV1.model_validate(item)) + return receipts + + +def get_receipt( + session: Session, receipt_id: uuid.UUID +) -> Optional[ConformanceReceiptV1]: + """Fetch a receipt by id, or ``None``.""" + for receipt in _iter_receipts(session): + if receipt.receipt_id == receipt_id: + return receipt + return None + + +def list_receipts( + session: Session, artifact_digest: Optional[str] = None +) -> Sequence[ConformanceReceiptV1]: + """List receipts, optionally for one artifact digest, oldest-first.""" + receipts = [ + receipt + for receipt in _iter_receipts(session) + if artifact_digest is None + or _same_digest(receipt.artifact_digest, artifact_digest) + ] + return sorted(receipts, key=lambda receipt: receipt.created_at) + + +def row_to_receipt(row: Any) -> ConformanceReceiptV1: + """Rehydrate a receipt from a row, a stored dict, or a model.""" + if isinstance(row, ConformanceReceiptV1): + return row + if isinstance(row, dict): + return ConformanceReceiptV1.model_validate(row) + return ConformanceReceiptV1.model_validate(row.receipt_json) + + +def conformance_coverage( + session: Session, release_id: Optional[str] = None +) -> dict[str, Any]: + """A coverage view: receipt counts by status and by ``(os, arch)`` host. + + ``release_id`` filters to that release's stored receipts; without it every + stored receipt is summarized. The view is explicit: a platform with no + ``PASS`` receipt is reported as uncovered, not omitted. + """ + statement = select(AgentRelease) + rows = list(session.execute(statement).scalars().all()) + receipts: list[ConformanceReceiptV1] = [] + for row in rows: + if release_id is not None and row.release_id != release_id: + continue + for item in (row.release_json or {}).get(CONFORMANCE_KEY) or []: + receipts.append(ConformanceReceiptV1.model_validate(item)) + + by_status: dict[str, int] = {} + by_host: dict[str, dict[str, int]] = {} + for receipt in receipts: + status = receipt.status.value + by_status[status] = by_status.get(status, 0) + 1 + host = by_host.setdefault(receipt.host.key, {}) + host[status] = host.get(status, 0) + 1 + return { + "release_id": release_id, + "total": len(receipts), + "by_status": by_status, + "by_host": { + host: { + "pass": counts.get("PASS", 0), + "fail": counts.get("FAIL", 0), + "unsupported": counts.get("UNSUPPORTED", 0), + "unknown": counts.get("UNKNOWN", 0), + "blocked": counts.get("BLOCKED", 0), + } + for host, counts in by_host.items() + }, + } diff --git a/src/research/study/packaging/verifier.py b/src/research/study/packaging/verifier.py new file mode 100644 index 0000000..0dbfbc3 --- /dev/null +++ b/src/research/study/packaging/verifier.py @@ -0,0 +1,340 @@ +"""Package verification and path containment (Issue 11). + +``verify_package`` is default-deny: it enforces relative safe paths that stay +under the package root, SHA-256 and size integrity, executability expectations, +no undeclared executables, no secret files, an optional HMAC over the manifest, +and an optional self-check. Every failure is typed; a package that cannot be +fully verified is never considered usable. +""" + +from __future__ import annotations + +import hashlib +import hmac +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +from .enums import PackageReasonCode +from .models import ( + PackageIssue, + PackageVerificationResult, + RuntimeManifestV2, + normalize_sha256, +) + +__all__ = [ + "EXECUTABLE_SUFFIXES", + "SECRET_FILE_PATTERNS", + "PathContainment", + "hash_file", + "resolve_under_root", + "scan_secret_files", + "scan_undeclared_executables", + "verify_package", +] + +#: File suffixes treated as executables even without a POSIX execute bit. +EXECUTABLE_SUFFIXES = frozenset({".exe", ".bat", ".cmd", ".com", ".sh", ".ps1", ".command"}) + +#: Name components that must never appear in a participant package. +SECRET_FILE_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + r"(^|[._-])\.env(\.[a-z0-9]+)?$", + r"(^|[._-])id_(rsa|dsa|ecdsa|ed25519)($|[._-])", + r"(^|[._-])(credentials|credential|secret|secrets|token|tokens)($|[._-])", + r"(^|[._-])\.(npmrc|netrc|pypirc|git-credentials)$", + r"\.(pem|key|p12|pfx|jks|keystore)$", + ) +) + +#: High-confidence secret *content* patterns for small text files. +SECRET_CONTENT_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----", + r"\bsk-[A-Za-z0-9]{16,}", + r"\bghp_[A-Za-z0-9]{20,}", + r"\bxox[baprs]-[A-Za-z0-9-]{10,}", + r"AWS_SECRET_ACCESS_KEY\s*=", + ) +) + +_TEXT_SUFFIXES = frozenset( + {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".txt", ".md", ".env", ".sh"} +) +_MAX_CONTENT_SCAN_BYTES = 256 * 1024 + +_SIGNATURE_PREFIXES = ("hmac-sha256:", "sha256:") + + +@dataclass +class PathContainment: + """Result of resolving a declared relative path under a trusted root.""" + + path: Optional[Path] = None + issue: Optional[PackageIssue] = None + errors: list[PackageIssue] = field(default_factory=list) + + @property + def ok(self) -> bool: + """Whether the path is contained and safe.""" + return self.path is not None and self.issue is None + + +def _issue(code: PackageReasonCode, message: str, field_name: str = "") -> PackageIssue: + return PackageIssue(code=code, message=message, field=field_name) + + +def resolve_under_root(root: Path, relative_path: str) -> PathContainment: + """Resolve ``relative_path`` under ``root``, rejecting escapes and symlink escapes.""" + if not relative_path or not relative_path.strip(): + return PathContainment(issue=_issue(PackageReasonCode.PATH_UNSAFE, "path must not be blank", relative_path)) + if "\x00" in relative_path: + return PathContainment(issue=_issue(PackageReasonCode.PATH_UNSAFE, "path must not contain NUL", relative_path)) + candidate = Path(relative_path) + if candidate.is_absolute(): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"absolute path '{relative_path}' is not allowed", relative_path)) + if relative_path.startswith("~"): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"home-relative path '{relative_path}' is not allowed", relative_path)) + if any(part == ".." for part in candidate.parts): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' contains '..'", relative_path)) + + resolved_root = root.resolve() + resolved = (resolved_root / candidate).resolve() + if resolved != resolved_root and resolved_root not in resolved.parents: + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' escapes the package root", relative_path)) + if resolved.exists() and resolved.is_symlink(): + real = resolved.resolve() + if real != resolved_root and resolved_root not in real.parents: + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' escapes the package root via a link", relative_path)) + return PathContainment(path=resolved) + + +def hash_file(path: Path) -> str: + """Stream a file and return its ``sha256:`` digest.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _is_executable_file(path: Path) -> bool: + if not path.is_file(): + return False + if path.suffix.lower() in EXECUTABLE_SUFFIXES: + return True + return hasattr(os, "X_OK") and os.access(path, os.X_OK) + + +def scan_undeclared_executables(root: Path, declared: set[str]) -> list[str]: + """Return executable-looking files under ``root`` that no component declares.""" + declared_normalized = {_normalize_relative(value) for value in declared} + undeclared: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or not _is_executable_file(path): + continue + relative = _normalize_relative(str(path.relative_to(root))) + if relative not in declared_normalized: + undeclared.append(relative) + return undeclared + + +def scan_secret_files(root: Path) -> list[str]: + """Return files under ``root`` whose name or content looks like a secret.""" + findings: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + relative = str(path.relative_to(root)) + if any(part.startswith((".aws", ".ssh", ".gnupg")) for part in path.parts): + findings.append(relative) + continue + if any(pattern.search(path.name.lower()) for pattern in SECRET_FILE_PATTERNS): + findings.append(relative) + continue + if path.suffix.lower() in _TEXT_SUFFIXES and path.stat().st_size <= _MAX_CONTENT_SCAN_BYTES: + try: + content = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + if any(pattern.search(content) for pattern in SECRET_CONTENT_PATTERNS): + findings.append(relative) + return findings + + +def _normalize_relative(value: str) -> str: + return value.replace("\\", "/").lstrip("./") + + +def _signature_payload(manifest: RuntimeManifestV2) -> bytes: + from research.canonical import canonical_bytes + + return canonical_bytes(manifest.canonical_payload()) + + +def _verify_signature( + manifest: RuntimeManifestV2, secret: Optional[str] +) -> Optional[PackageIssue]: + if not manifest.signature: + return None # a signature is optional; when absent nothing is claimed + if not secret: + return _issue( + PackageReasonCode.SIGNATURE_MISMATCH, + "manifest claims a signature but no verification secret was supplied", + "signature", + ) + expected = hmac.new(secret.encode("utf-8"), _signature_payload(manifest), hashlib.sha256).hexdigest() + claimed = manifest.signature.strip().lower() + for prefix in _SIGNATURE_PREFIXES: + if claimed.startswith(prefix): + claimed = claimed[len(prefix) :] + break + if not hmac.compare_digest(claimed, expected): + return _issue( + PackageReasonCode.SIGNATURE_MISMATCH, + "manifest signature does not match the canonical payload", + "signature", + ) + return None + + +def sign_manifest(manifest: RuntimeManifestV2, secret: str) -> str: + """Return the HMAC signature over the manifest's canonical payload.""" + return "hmac-sha256:" + hmac.new( + secret.encode("utf-8"), _signature_payload(manifest), hashlib.sha256 + ).hexdigest() + + +def verify_package( + root: Path, + manifest: RuntimeManifestV2, + secret: Optional[str] = None, + now: Optional[object] = None, + *, + self_check_runner: Optional[Callable[[RuntimeManifestV2, Path], int]] = None, +) -> PackageVerificationResult: + """Verify a package directory against ``manifest``. Default-deny.""" + del now # validation is time-independent; kept for a stable call signature + errors: list[PackageIssue] = [] + + if manifest.schema_version != "2": + errors.append( + _issue( + PackageReasonCode.SCHEMA_VERSION_UNSUPPORTED, + f"manifest schema_version '{manifest.schema_version}' is not supported", + "schema_version", + ) + ) + if not manifest.digest_matches(): + errors.append( + _issue(PackageReasonCode.DIGEST_MISMATCH, "manifest content does not match manifest_digest", "manifest_digest") + ) + if not manifest.components: + errors.append(_issue(PackageReasonCode.EMPTY_COMPONENTS, "manifest declares no components", "components")) + + signature_issue = _verify_signature(manifest, secret) + if signature_issue is not None: + errors.append(signature_issue) + + if not root.is_dir(): + errors.append(_issue(PackageReasonCode.ARTIFACT_MISSING, f"package root '{root}' is not a directory", "root")) + return PackageVerificationResult(valid=False, manifest_digest=manifest.manifest_digest, errors=errors) + + verified = 0 + declared_paths: set[str] = set() + for component in manifest.components: + containment = resolve_under_root(root, component.relative_path) + if not containment.ok: + errors.append( + containment.issue + or _issue(PackageReasonCode.PATH_ESCAPE, "component path is not contained", component.name) + ) + continue + declared_paths.add(_normalize_relative(component.relative_path)) + path = containment.path + assert path is not None + if not path.is_file(): + errors.append( + _issue(PackageReasonCode.ARTIFACT_MISSING, f"component '{component.name}' file is missing", component.relative_path) + ) + continue + actual_size = path.stat().st_size + if actual_size != component.size: + errors.append( + _issue( + PackageReasonCode.SIZE_MISMATCH, + f"component '{component.name}' size {actual_size} != declared {component.size}", + component.relative_path, + ) + ) + continue + if normalize_sha256(hash_file(path)) != normalize_sha256(component.sha256): + errors.append( + _issue(PackageReasonCode.DIGEST_MISMATCH, f"component '{component.name}' digest mismatch", component.relative_path) + ) + continue + if component.executable and not _is_executable_file(path): + errors.append( + _issue(PackageReasonCode.NOT_EXECUTABLE, f"component '{component.name}' is declared executable but is not", component.relative_path) + ) + continue + verified += 1 + + for relative in scan_undeclared_executables(root, declared_paths): + errors.append( + _issue(PackageReasonCode.UNDECLARED_EXECUTABLE, f"executable '{relative}' is not declared by any component", relative) + ) + for relative in scan_secret_files(root): + errors.append( + _issue(PackageReasonCode.SECRET_FILE_PRESENT, f"secret-shaped file '{relative}' must not be packaged", relative) + ) + + if manifest.self_check is not None: + errors.extend(_verify_self_check(root, manifest, self_check_runner)) + + return PackageVerificationResult( + valid=not errors, + manifest_digest=manifest.manifest_digest, + verified_components=verified, + errors=errors, + ) + + +def _verify_self_check( + root: Path, + manifest: RuntimeManifestV2, + runner: Optional[Callable[[RuntimeManifestV2, Path], int]], +) -> list[PackageIssue]: + spec = manifest.self_check + assert spec is not None + if not spec.command: + return [_issue(PackageReasonCode.SELF_CHECK_FAILED, "self-check declares an empty command", "self_check.command")] + command_path = resolve_under_root(root, spec.command[0]) + if not command_path.ok or command_path.path is None or not command_path.path.exists(): + return [ + _issue( + PackageReasonCode.SELF_CHECK_FAILED, + f"self-check executable '{spec.command[0]}' is not present under the package root", + "self_check.command", + ) + ] + if runner is None: + return [] + try: + exit_code = runner(manifest, command_path.path) + except Exception as error: # noqa: BLE001 - a throwing self-check is a failure + return [_issue(PackageReasonCode.SELF_CHECK_FAILED, f"self-check raised: {error}", "self_check")] + if exit_code != spec.expected_exit_code: + return [ + _issue( + PackageReasonCode.SELF_CHECK_FAILED, + f"self-check exited {exit_code}, expected {spec.expected_exit_code}", + "self_check", + ) + ] + return [] diff --git a/src/research/study/protocol/__init__.py b/src/research/study/protocol/__init__.py new file mode 100644 index 0000000..530efad --- /dev/null +++ b/src/research/study/protocol/__init__.py @@ -0,0 +1,122 @@ +"""Versioned study protocol and immutable publication (Issue 02). + +Public surface: + +* :mod:`research.study.protocol.enums` - closed vocabularies for revision status, + schedule kind, assignment unit/strategy, retention action, telemetry field + classes, completion policy, validation reason codes, release-resolution + statuses and publication outcomes. +* :mod:`research.study.protocol.models` - Pydantic v2 contracts for + ``StudyProtocolV1`` and its versioned sub-objects, including the explicit + :class:`~research.study.protocol.models.ExplicitUnknown` state. +* :mod:`research.study.protocol.canonical` - deterministic, order-normalized JSON and + SHA-256 digest of a validated protocol (reusing the Issue 01 canonical + helpers). +* :mod:`research.study.protocol.validation` - typed field-level, cross-reference and + document-safety validation, plus the injected + :class:`~research.study.protocol.validation.ReleaseResolver` interface and its + :class:`~research.study.protocol.validation.NullReleaseResolver` default. +* :mod:`research.study.protocol.publication` - immutable revision creation, revision + lineage, optimistic-concurrency conflicts, retirement and audit records. + +The persistence adapters live in :mod:`research.study.protocol.store` and take a +caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" + +from .canonical import protocol_canonical_json, protocol_digest +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + PublicationOutcome, + ReleaseResolutionStatus, + RetentionAction, + RevisionStatus, + ScheduleKind, + TelemetryFieldClass, + ValidationReasonCode, + ValidationSeverity, +) +from .models import ( + ExplicitUnknown, + FixedSchedule, + ResolvedDistribution, + RollingSchedule, + StudyProtocolV1, +) +from .publication import ( + AuditRecord, + ImmutableRevisionError, + PublicationResult, + RetirementResult, + RevisionConflict, + RevisionLineage, + StudyRevision, + assert_revision_mutable, + lineage_from_revisions, + publish_revision, + retire_revision, +) +from .validation import ( + DistributionResolution, + DistributionResolver, + NullDistributionResolver, + NullReleaseResolver, + ProtocolValidationError, + ReleaseResolution, + ReleaseResolver, + ValidationError, + blocking_errors, + freeze_protocol_distributions, + is_publishable, + normalized_condition_weights, + validate_protocol, + warnings, +) + +__all__ = [ + "AssignmentStrategy", + "AssignmentUnit", + "AuditRecord", + "CompletionPolicyKind", + "DistributionResolution", + "DistributionResolver", + "ExplicitUnknown", + "FixedSchedule", + "ImmutableRevisionError", + "NullDistributionResolver", + "NullReleaseResolver", + "ProtocolValidationError", + "PublicationOutcome", + "PublicationResult", + "ReleaseResolution", + "ReleaseResolutionStatus", + "ReleaseResolver", + "ResolvedDistribution", + "RetentionAction", + "RevisionConflict", + "RevisionLineage", + "RevisionStatus", + "RetirementResult", + "RollingSchedule", + "ScheduleKind", + "StudyProtocolV1", + "StudyRevision", + "TelemetryFieldClass", + "ValidationError", + "ValidationReasonCode", + "ValidationSeverity", + "assert_revision_mutable", + "blocking_errors", + "freeze_protocol_distributions", + "is_publishable", + "lineage_from_revisions", + "normalized_condition_weights", + "protocol_canonical_json", + "protocol_digest", + "publish_revision", + "retire_revision", + "validate_protocol", + "warnings", +] diff --git a/src/research/study/protocol/canonical.py b/src/research/study/protocol/canonical.py new file mode 100644 index 0000000..2943a5b --- /dev/null +++ b/src/research/study/protocol/canonical.py @@ -0,0 +1,159 @@ +"""Deterministic canonical serialization and SHA-256 digest for study protocols. + +This module reuses the Issue 01 canonical helpers +(:mod:`research.compatibility.canonical`) rather than reimplementing JSON +sorting and hashing. It adds one protocol-specific concern: normalizing +*set-like* arrays so that an equivalent document hashes identically no matter +how its lists were authored. + +Canonicalization rules +---------------------- + +* Map keys are sorted (delegated to ``canonical_json``). +* ``conditions`` are sorted by ``condition_id``; condition order is not + meaningful for execution because assignment selects by id and weight. +* ``environment_requirements.required_capabilities`` are sorted by + ``(capability, require_state)``, ``consent.locale_refs`` and + ``assignment.stratification`` are sorted lexically, and ``survey_hooks`` are + sorted by ``(survey_id, trigger)``. +* A condition's unset ``resolved_distribution`` is omitted: it is absent on a + draft and populated by the server at publication, so omitting it keeps a + draft's digest independent of an optional pin that is not part of the draft. +* Everything else is preserved as-is, including ``None`` (inherited/default) + and :class:`~research.study.protocol.models.ExplicitUnknown` values. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from research.compatibility.canonical import ( # re-exported for callers + canonical_bytes, + canonical_hash, + canonical_json, +) + +if TYPE_CHECKING: + from .models import StudyProtocolV1 + +__all__ = [ + "canonical_bytes", + "canonical_hash", + "canonical_json", + "canonicalize_protocol", + "protocol_canonical_bytes", + "protocol_canonical_json", + "protocol_digest", +] + + +def _sorted_list(values: Any, key) -> list[Any]: + if not isinstance(values, list): + return [] + return sorted(values, key=key) + + +def canonicalize_protocol(protocol: StudyProtocolV1) -> dict[str, Any]: + """Return the canonical, order-normalized mapping for ``protocol``.""" + data = protocol.model_dump(mode="json") + + conditions = data.get("conditions") + if isinstance(conditions, list): + data["conditions"] = sorted( + conditions, + key=lambda item: ( + str(item.get("condition_id", "")), + str(item.get("name") or ""), + ), + ) + # An unset frozen pin is semantically absent on a draft, so it is + # omitted from the canonical bytes rather than serialized as ``null``. + for item in data["conditions"]: + if not isinstance(item, dict): + continue + if item.get("resolved_distribution") is None: + item.pop("resolved_distribution", None) + + environment = data.get("environment_requirements") + if isinstance(environment, dict): + environment["required_capabilities"] = _sorted_list( + environment.get("required_capabilities"), + key=lambda item: ( + str(item.get("capability", "")), + str(item.get("require_state", "")), + ), + ) + + telemetry = data.get("telemetry_policy") + if isinstance(telemetry, dict): + telemetry["allowed_field_classes"] = _sorted_list( + telemetry.get("allowed_field_classes"), key=str + ) + + assignment = data.get("assignment") + if isinstance(assignment, dict) and isinstance( + assignment.get("stratification"), list + ): + assignment["stratification"] = _sorted_list( + assignment["stratification"], key=str + ) + + survey_hooks = data.get("survey_hooks") + if isinstance(survey_hooks, list): + data["survey_hooks"] = sorted( + survey_hooks, + key=lambda item: ( + str(item.get("survey_id", "")), + str(item.get("trigger", "")), + ), + ) + + governance = data.get("schema_governance") + if isinstance(governance, dict): + governance["compatible_from_schema_versions"] = _sorted_list( + governance.get("compatible_from_schema_versions"), key=str + ) + + return data + + +def _digest_mapping(protocol: StudyProtocolV1) -> dict[str, Any]: + """Canonical mapping with resolution metadata excluded from the digest. + + ``resolved_at`` records *when* a pin was resolved, not the pin's semantic + identity, so it is dropped from the digest preimage: re-publishing identical + content stays content-addressed regardless of wall-clock resolution time. + The timestamp is still persisted in the stored ``protocol_json``. + """ + data = canonicalize_protocol(protocol) + for item in data.get("conditions", []) or []: + if not isinstance(item, dict): + continue + frozen = item.get("resolved_distribution") + if isinstance(frozen, dict): + frozen.pop("resolved_at", None) + # An absent frozen profile config is semantically absent (legacy + # revisions), so it must not change a document's digest. + if frozen.get("agent_config") is None: + frozen.pop("agent_config", None) + return data + + +def protocol_canonical_json(protocol: StudyProtocolV1) -> str: + """Canonical JSON string for a protocol (sorted keys, compact separators).""" + return canonical_json(_digest_mapping(protocol)) + + +def protocol_canonical_bytes(protocol: StudyProtocolV1) -> bytes: + """UTF-8 encoded :func:`protocol_canonical_json`.""" + return protocol_canonical_json(protocol).encode("utf-8") + + +def protocol_digest(protocol: StudyProtocolV1) -> str: + """SHA-256 hex digest of the canonical protocol document.""" + return canonical_hash(_digest_mapping(protocol)) + + +def digest_matches(protocol: StudyProtocolV1, expected_digest: str) -> bool: + """Whether ``expected_digest`` equals the canonical digest of ``protocol``.""" + return bool(expected_digest) and protocol_digest(protocol) == expected_digest diff --git a/src/research/study/protocol/enums.py b/src/research/study/protocol/enums.py new file mode 100644 index 0000000..ea15cc8 --- /dev/null +++ b/src/research/study/protocol/enums.py @@ -0,0 +1,179 @@ +"""Closed vocabularies for the versioned study protocol (Issue 02). + +Every value here is part of a persisted contract (draft JSON, published +revision JSON, API responses), so members are additive only: renaming or +removing a member is a breaking change to already-stored revisions. +""" + +from __future__ import annotations + +from enum import Enum + + +class SchemaVersion(str, Enum): + """Supported ``StudyProtocolV1.schema_version`` values.""" + + V1 = "1" + + +class RevisionStatus(str, Enum): + """Lifecycle of a study revision. + + ``PUBLISHED`` is immutable in content; only ``RETIRED`` is a permitted + lifecycle transition, and it never changes the stored protocol bytes. + """ + + DRAFT = "DRAFT" + PUBLISHED = "PUBLISHED" + RETIRED = "RETIRED" + + +class ScheduleKind(str, Enum): + """Whether a study runs to a fixed calendar window or a rolling duration.""" + + FIXED = "FIXED" + ROLLING = "ROLLING" + + +class AssignmentUnit(str, Enum): + """Unit that is randomized by the assignment service. + + Issue 02 only supports enrollment-level assignment. ``SESSION`` exists as an + additive contract value so a future protocol version can request it without + a breaking enum change; the V1 validator rejects it explicitly. + """ + + ENROLLMENT = "ENROLLMENT" + SESSION = "SESSION" + + +class AssignmentStrategy(str, Enum): + """How an assignment service picks a condition for one unit.""" + + WEIGHTED_RANDOM = "WEIGHTED_RANDOM" + DETERMINISTIC_HASH = "DETERMINISTIC_HASH" + STRATIFIED = "STRATIFIED" + + +class RetentionAction(str, Enum): + """What happens to identifiable data when retention expires.""" + + RETAIN_ANONYMIZED = "RETAIN_ANONYMIZED" + DELETE_IDENTIFIABLE = "DELETE_IDENTIFIABLE" + DELETE_ALL = "DELETE_ALL" + + +class TelemetryFieldClass(str, Enum): + """Coarse class of telemetry fields the protocol permits capturing.""" + + STRUCTURAL = "STRUCTURAL" + CONTENT = "CONTENT" + METRICS = "METRICS" + DIAGNOSTICS = "DIAGNOSTICS" + + +class CompletionPolicyKind(str, Enum): + """Terminal condition that ends enrollment for a study.""" + + MANUAL = "MANUAL" + TARGET_CAPACITY = "TARGET_CAPACITY" + SCHEDULE_END = "SCHEDULE_END" + + +class ValidationSeverity(str, Enum): + """Whether a validation reason blocks publication, asks for review, or warns. + + ``ERROR`` blocks publication. ``NEEDS_REVIEW`` asks an operator to look + before proceeding and blocks an automatic publish. ``WARNING`` is + informational only: the document may publish, but the caller is expected to + surface the reason (for example an administrator selecting an unverified + distribution). + """ + + ERROR = "ERROR" + NEEDS_REVIEW = "NEEDS_REVIEW" + WARNING = "WARNING" + + +class ValidationReasonCode(str, Enum): + """Stable machine-readable reason a protocol cannot be published.""" + + # Schema / structural. + SCHEMA_INVALID = "SCHEMA_INVALID" + UNSUPPORTED_SCHEMA_VERSION = "UNSUPPORTED_SCHEMA_VERSION" + NO_CONDITIONS = "NO_CONDITIONS" + + # Conditions / weights. + DUPLICATE_CONDITION_ID = "DUPLICATE_CONDITION_ID" + NON_POSITIVE_WEIGHT = "NON_POSITIVE_WEIGHT" + EMPTY_WEIGHT_TOTAL = "EMPTY_WEIGHT_TOTAL" + + # Schedule. + FIXED_SCHEDULE_EXPIRED = "FIXED_SCHEDULE_EXPIRED" + FIXED_SCHEDULE_INVALID = "FIXED_SCHEDULE_INVALID" + ROLLING_DURATION_INVALID = "ROLLING_DURATION_INVALID" + + # Assignment. + ASSIGNMENT_UNIT_NOT_ENROLLMENT = "ASSIGNMENT_UNIT_NOT_ENROLLMENT" + ASSIGNMENT_STRATEGY_UNKNOWN = "ASSIGNMENT_STRATEGY_UNKNOWN" + ASSIGNMENT_STRATIFICATION_MISSING = "ASSIGNMENT_STRATIFICATION_MISSING" + + # Consent / privacy. + CONSENT_DIGEST_MISSING = "CONSENT_DIGEST_MISSING" + # Unsupported retention authoring: the published policy retains uploaded + # data according to consent; a participant-selected DELETE_ALL is not an + # authorable option. + RETENTION_UNSUPPORTED = "RETENTION_UNSUPPORTED" + + # Agent release references. + AGENT_RELEASE_UNPINNED = "AGENT_RELEASE_UNPINNED" + AGENT_RELEASE_LATEST = "AGENT_RELEASE_LATEST" + MUTABLE_AGENT_COMMAND = "MUTABLE_AGENT_COMMAND" + RELEASE_UNRESOLVED = "RELEASE_UNRESOLVED" + RELEASE_UNQUALIFIED = "RELEASE_UNQUALIFIED" + RELEASE_WITHDRAWN = "RELEASE_WITHDRAWN" + RELEASE_DIGEST_MISMATCH = "RELEASE_DIGEST_MISMATCH" + + # Agent profile references (a condition may pin an existing profile). + AGENT_PROFILE_NOT_FOUND = "AGENT_PROFILE_NOT_FOUND" + + # A distribution (AgentProfile) is not verified: its release is missing a + # passing conformance receipt, or it is a participant-installed BYOA + # distribution that has no artifact to bind conformance evidence to. + DISTRIBUTION_UNVERIFIED = "DISTRIBUTION_UNVERIFIED" + + # Declared distribution contract on a release pin. + AGENT_RELEASE_MODE_UNKNOWN = "AGENT_RELEASE_MODE_UNKNOWN" + # A BYOA pin must carry a release identity (release_id/version), never a digest. + AGENT_RELEASE_IDENTITY_REQUIRED = "AGENT_RELEASE_IDENTITY_REQUIRED" + + # Environment requirements. + UNKNOWN_REQUIRED_CAPABILITY = "UNKNOWN_REQUIRED_CAPABILITY" + UNKNOWN_ENVIRONMENT_REQUIREMENT = "UNKNOWN_ENVIRONMENT_REQUIREMENT" + ENVIRONMENT_PROTOCOL_VERSION_MISSING = "ENVIRONMENT_PROTOCOL_VERSION_MISSING" + + # Telemetry / completion. + TELEMETRY_FIELD_CLASSES_EMPTY = "TELEMETRY_FIELD_CLASSES_EMPTY" + COMPLETION_TARGET_INVALID = "COMPLETION_TARGET_INVALID" + + # Document safety. + FORBIDDEN_IDENTIFIER = "FORBIDDEN_IDENTIFIER" + FORBIDDEN_CREDENTIAL = "FORBIDDEN_CREDENTIAL" + FORBIDDEN_LOCAL_PATH = "FORBIDDEN_LOCAL_PATH" + + +class ReleaseResolutionStatus(str, Enum): + """Outcome of resolving a pinned agent release through a resolver.""" + + RESOLVED = "RESOLVED" + NOT_FOUND = "NOT_FOUND" + UNQUALIFIED = "UNQUALIFIED" + WITHDRAWN = "WITHDRAWN" + + +class PublicationOutcome(str, Enum): + """Terminal result of a publication attempt.""" + + PUBLISHED = "PUBLISHED" + CONFLICT = "CONFLICT" + VALIDATION_FAILED = "VALIDATION_FAILED" diff --git a/src/research/study/protocol/join_code.py b/src/research/study/protocol/join_code.py new file mode 100644 index 0000000..a6e1032 --- /dev/null +++ b/src/research/study/protocol/join_code.py @@ -0,0 +1,74 @@ +"""Human-shareable study join codes (researcher -> participant onboarding). + +A join code is a short, random, study-scoped handle that a researcher shares +with prospective participants. It is bound to exactly one *published* revision: +redeeming it enrolls the caller in that revision's study without ever letting +the client choose a revision, an eligibility verdict, or a consent document. + +Design constraints: + +* **Opaque and random.** The code is drawn from a CSPRNG and carries no study, + revision or account semantics. It can therefore be printed, spoken or typed + without leaking anything beyond "this study exists". +* **Human-typable.** The alphabet is Crockford base32 (no ``I``/``L``/``O``/``U``) + so an 8-character code is unambiguous to read aloud and to type. Normalization + folds the common look-alikes back to their canonical character. +* **Unique per published revision.** The database enforces uniqueness; the + in-process allocator retries on the astronomically unlikely collision. +* **Published-only.** Drafts never get a code, so an unpublished protocol can + never be joined. +""" + +from __future__ import annotations + +import re +import secrets + +__all__ = [ + "JOIN_CODE_ALPHABET", + "JOIN_CODE_LENGTH", + "generate_join_code", + "is_well_formed_join_code", + "normalize_join_code", +] + +#: Crockford base32: 0-9 and A-Z minus I, L, O and U. +JOIN_CODE_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + +#: Default code length. 32**8 == 2**40 possible codes. +JOIN_CODE_LENGTH = 8 + +_NON_CODE_CHARS = re.compile(r"[^0-9A-Z]") + +# Common look-alikes a human may type instead of the canonical character. +_LOOKALIKES = str.maketrans({"I": "1", "L": "1", "O": "0", "U": "V"}) + + +def generate_join_code(length: int = JOIN_CODE_LENGTH) -> str: + """Return a fresh random join code. + + The code is drawn from :data:`JOIN_CODE_ALPHABET` using a CSPRNG. It is not + derived from a study, revision or account identifier. + """ + if length <= 0: + raise ValueError("join code length must be positive") + return "".join(secrets.choice(JOIN_CODE_ALPHABET) for _ in range(length)) + + +def normalize_join_code(code: str) -> str: + """Return the canonical form of a user-supplied join code. + + Whitespace, separators and lower case are folded away, and Crockford + look-alikes (``I``/``L`` -> ``1``, ``O`` -> ``0``, ``U`` -> ``V``) are mapped + to their canonical character. An empty/garbage input normalizes to the empty + string, which never resolves. + """ + if not code: + return "" + upper = code.strip().upper().translate(_LOOKALIKES) + return _NON_CODE_CHARS.sub("", upper) + + +def is_well_formed_join_code(code: str, length: int = JOIN_CODE_LENGTH) -> bool: + """Whether ``code`` (already normalized) is exactly a join code.""" + return len(code) == length and all(char in JOIN_CODE_ALPHABET for char in code) diff --git a/src/research/study/protocol/models.py b/src/research/study/protocol/models.py new file mode 100644 index 0000000..701241d --- /dev/null +++ b/src/research/study/protocol/models.py @@ -0,0 +1,318 @@ +"""Pydantic v2 contracts for the versioned study protocol (``StudyProtocolV1``). + +Semantics that matter for reproducibility: + +* ``None`` means an *intentional inherited/default* value. It is never coerced + to a concrete value, and a missing field dumps back as ``None``. +* :class:`ExplicitUnknown` is the typed *insufficient evidence* state. It is a + distinct value from ``None`` and is never normalized to a default. +* Condition order is not meaningful for execution, so the canonical serializer + sorts conditions (and other set-like collections) by stable identity before + hashing. See :mod:`research.study.protocol.canonical`. + +The document is draft-only metadata plus policy. It must never carry a +participant/session/account identifier or a credential; those are rejected by +:mod:`research.study.protocol.validation`. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Literal, Optional, Union +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + RetentionAction, + ScheduleKind, + TelemetryFieldClass, +) + +# Document-wide policy: unknown keys are rejected so a typo cannot silently +# drop a policy. Draft content is validated before it is ever persisted. +_BASE_CONFIG = ConfigDict(extra="forbid") + + +class ExplicitUnknown(BaseModel): + """Typed "insufficient evidence" marker. + + Using a model (rather than ``None`` or the string ``"unknown"``) means an + unknown value survives canonical serialization and round-trips as a + distinct state, so a consumer can never mistake it for an inherited default. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["UNKNOWN"] = "UNKNOWN" + + +class StudyMetadata(BaseModel): + """Draft-only, non-secret descriptive metadata.""" + + model_config = _BASE_CONFIG + + name: str + description: Optional[str] = None + owner: Optional[str] = None + + +class FixedSchedule(BaseModel): + """A study that accepts data between two absolute instants.""" + + model_config = _BASE_CONFIG + + kind: Literal[ScheduleKind.FIXED] = ScheduleKind.FIXED + start_at: datetime + end_at: Optional[datetime] = None + + +class RollingSchedule(BaseModel): + """A study whose window is a duration measured from enrollment.""" + + model_config = _BASE_CONFIG + + kind: Literal[ScheduleKind.ROLLING] = ScheduleKind.ROLLING + # Required and strictly positive for publication; ``None`` is an explicit + # missing value rather than an implicit default. + duration_seconds: Optional[int] = None + + +Schedule = Union[FixedSchedule, RollingSchedule] + + +class EnrollmentPolicy(BaseModel): + """How many units the study accepts and whether re-entry is allowed.""" + + model_config = _BASE_CONFIG + + # ``None`` = inherit the server default capacity; ExplicitUnknown = capacity + # is not yet known and must be resolved before publication. + capacity: Optional[Union[int, ExplicitUnknown]] = None + allow_reentry: Optional[bool] = None + + +class AssignmentPolicy(BaseModel): + """How the assignment service maps an enrollment unit to a condition. + + Assignments are always sticky: an existing assignment for the same + ``(enrollment_id, study_revision_id)`` is reused and never re-randomized. + There is no runtime toggle. Reallocation, if ever required, must be defined + by a successor revision rather than by mutating an existing assignment. + """ + + model_config = _BASE_CONFIG + + unit: AssignmentUnit = AssignmentUnit.ENROLLMENT + # Free string, not the enum: an unknown strategy must be preserved so the + # validator can reject it with a typed reason instead of failing to parse. + strategy: str = AssignmentStrategy.WEIGHTED_RANDOM.value + # Required when ``strategy`` is STRATIFIED; stable stratum key names. + stratification: Optional[list[str]] = None + + +class ResolvedAgentConfig(BaseModel): + """The frozen, non-secret execution config of a condition's profile. + + Publication copies these values into the revision so a later profile edit + changes neither historical display nor a published study's task/bootstrap + configuration. It deliberately contains no endpoint URL, secret reference or + secret value. + """ + + model_config = _BASE_CONFIG + + profile_id: UUID + name: str + model: str + framework_version: str + tools_json: str + approval_policy: str + max_steps: int + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + connection_id: Optional[UUID] = None + connection_label: Optional[str] = None + # The researcher who funds the connection grant for this condition: the + # study owner (``Study.created_by``) at publication. Non-secret and never an + # email/name, so it is safe to freeze and to include in the digest. + funding_owner_user_id: Optional[UUID] = None + + +class ResolvedDistribution(BaseModel): + """The distribution pin, frozen and written at publication. + + A draft condition only names an ``AgentProfile`` (a *distribution*) by its + opaque ``distribution_id``. Because distributions are mutable, admin-editable + rows, a published revision freezes the exact resolved pin here so an + immutable revision can always answer "which artifact produced this data?". + + ``PACKAGED`` distributions resolve to a digest-pinned registry release; a + participant-installed ``BYOA_EXTERNAL`` distribution carries the command / + package identity and no digest. ``verified`` is derived at resolution time + and is informational once frozen (the frozen value is never re-derived). + """ + + model_config = _BASE_CONFIG + + distribution_id: UUID + distribution_mode: str = "PACKAGED" + release_id: Optional[str] = None + agent_id: Optional[str] = None + version: Optional[str] = None + agent_package: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + artifact_digest: Optional[str] = None + verified: bool = False + resolved_at: Optional[datetime] = None + # Full, non-secret profile config frozen at publication. + agent_config: Optional[ResolvedAgentConfig] = None + + +class StudyCondition(BaseModel): + """One experimental arm: its weight and the distribution it runs. + + ``distribution_id`` names a server-side ``AgentProfile`` by opaque id only. + The profile is the *distribution*: it holds the provider ``base_url``, the + ``api_key_ref`` *name* (never the secret), the model and the artifact pin + (``release_id`` for ``PACKAGED``; ``agent_package``/``agent_command`` for + ``BYOA_EXTERNAL``). None of that is duplicated into the draft document. + + ``resolved_distribution`` is absent on a draft and is populated at + publication with the frozen, resolved pin (never authored by a caller). + """ + + model_config = _BASE_CONFIG + + condition_id: str + name: Optional[str] = None + # Relative, strictly positive frequency. Weights are normalized + # deterministically at assignment time (see + # :func:`research.study.protocol.validation.normalized_condition_weights`). + weight: float + # Opaque reference to an existing AgentProfile/"distribution" (validated by + # the backend, which is the only layer with database access). Never a secret. + distribution_id: UUID + adapter_version: Optional[str] = None + # Condition-scoped, non-secret overrides. Null/absent means "inherit". + declared_overrides: dict[str, Any] = Field(default_factory=dict) + # Frozen at publish; never supplied by a caller and rejected if present on a + # draft that has not been resolved by the server. + resolved_distribution: Optional[ResolvedDistribution] = None + + +class SessionPolicy(BaseModel): + """Timeouts that govern an enrolled session's lifecycle.""" + + model_config = _BASE_CONFIG + + idle_timeout_seconds: Optional[int] = None + resume_grace_seconds: Optional[int] = None + heartbeat_seconds: Optional[int] = None + + +class TelemetryPolicy(BaseModel): + """Which classes of telemetry fields the study permits capturing.""" + + model_config = _BASE_CONFIG + + allowed_field_classes: list[TelemetryFieldClass] = Field(default_factory=list) + + +class PrivacyPolicy(BaseModel): + """Retention and deletion behavior for enrolled data.""" + + model_config = _BASE_CONFIG + + retention_action: RetentionAction + retention_days: Optional[int] = None + + +class CapabilityExpectation(BaseModel): + """One required host/agent capability predicate. + + ``capability`` is a free string rather than an enum so an unknown-but- + required capability is preserved and flagged ``NEEDS_REVIEW`` instead of + being silently dropped by parsing. + """ + + model_config = _BASE_CONFIG + + capability: str + require_state: str = "SUPPORTED" + evidence_ref: Optional[str] = None + + +class EnvironmentRequirements(BaseModel): + """Host/agent compatibility a revision must satisfy before launch.""" + + model_config = _BASE_CONFIG + + # Expected negotiated ACP protocol version. Required for publication. + expected_protocol_version: Optional[str] = None + required_capabilities: list[CapabilityExpectation] = Field(default_factory=list) + # Untested-but-expected environment facts; explicit unknown is allowed and + # blocks publication with NEEDS_REVIEW. + host_kind: Optional[Union[str, ExplicitUnknown]] = None + + +class CompletionPolicy(BaseModel): + """When the study stops accepting new enrollments.""" + + model_config = _BASE_CONFIG + + policy: CompletionPolicyKind = CompletionPolicyKind.MANUAL + target_enrollments: Optional[int] = None + + +class SchemaGovernance(BaseModel): + """How the protocol schema itself may change for this study.""" + + model_config = _BASE_CONFIG + + owner: Optional[str] = None + change_policy: Optional[str] = None + compatible_from_schema_versions: list[str] = Field(default_factory=list) + + +class SurveyHook(BaseModel): + """Optional, versioned survey attachment point.""" + + model_config = _BASE_CONFIG + + survey_id: str + trigger: str + version: Optional[str] = None + + +class StudyProtocolV1(BaseModel): + """The complete, validated experimental contract (schema version ``1``).""" + + model_config = _BASE_CONFIG + + schema_version: str = "1" + study_id: UUID + metadata: StudyMetadata + # Explicit discriminated union: a document must choose fixed vs rolling. + schedule: Schedule = Field(discriminator="kind") + enrollment: EnrollmentPolicy = Field(default_factory=EnrollmentPolicy) + assignment: AssignmentPolicy = Field(default_factory=AssignmentPolicy) + conditions: list[StudyCondition] = Field(default_factory=list) + session_policy: SessionPolicy = Field(default_factory=SessionPolicy) + telemetry_policy: TelemetryPolicy = Field(default_factory=TelemetryPolicy) + privacy_policy: PrivacyPolicy + # Deprecated, tolerated only so existing protocol fixtures and the website + # editor payload keep parsing: consent is now a single global text accepted + # once at join and is never stored, versioned or checked per study. Remove + # this field once the editor and fixtures stop sending `consent`. + consent: Optional[dict[str, Any]] = None + environment_requirements: EnvironmentRequirements = Field( + default_factory=EnvironmentRequirements + ) + completion: CompletionPolicy = Field(default_factory=CompletionPolicy) + schema_governance: Optional[SchemaGovernance] = None + survey_hooks: list[SurveyHook] = Field(default_factory=list) diff --git a/src/research/study/protocol/publication.py b/src/research/study/protocol/publication.py new file mode 100644 index 0000000..37de817 --- /dev/null +++ b/src/research/study/protocol/publication.py @@ -0,0 +1,283 @@ +"""Immutable publication and revision lineage for study protocols. + +Publication is a pure function of a validated draft plus the study's existing +revision lineage: + +* a new :class:`StudyRevision` is produced with a fresh ``revision_id``, a + monotonic ``revision_number``, canonical ``protocol_json`` and its SHA-256 + ``protocol_digest``; +* the prior revision is never mutated; the new revision records + ``supersedes_revision_id`` so lineage is auditable; +* publishing with a stale ``expected_revision_number`` returns a typed + :class:`RevisionConflict` instead of overwriting a concurrent revision; +* an audit record is emitted for every state transition. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from .canonical import canonicalize_protocol, protocol_digest +from .enums import PublicationOutcome, RevisionStatus +from .validation import ( + DistributionResolver, + ValidationError, + blocking_errors, + freeze_protocol_distributions, + validate_protocol, +) + +if TYPE_CHECKING: + from .models import StudyProtocolV1 + + +class StudyRevision(BaseModel): + """One immutable, content-addressed draft of a study's experimental intent.""" + + model_config = ConfigDict(extra="forbid") + + revision_id: UUID + study_id: UUID + revision_number: int + status: RevisionStatus + # Canonical mapping (set-like arrays normalized) of the protocol document. + protocol_json: dict[str, Any] + protocol_digest: str + published_at: Optional[datetime] = None + supersedes_revision_id: Optional[UUID] = None + created_at: Optional[datetime] = None + + +class RevisionLineage(BaseModel): + """The lineage head a publication call must extend.""" + + model_config = ConfigDict(extra="forbid") + + study_id: UUID + latest_revision_id: Optional[UUID] = None + latest_revision_number: int = 0 + latest_digest: Optional[str] = None + latest_status: Optional[RevisionStatus] = None + + +class RevisionConflict(BaseModel): + """Typed optimistic-concurrency conflict.""" + + model_config = ConfigDict(extra="forbid") + + expected_revision_number: int + actual_revision_number: int + message: str = "a newer revision was published concurrently" + + +class AuditRecord(BaseModel): + """Append-only record of a publication or lifecycle transition.""" + + model_config = ConfigDict(extra="forbid") + + event_type: str + study_id: UUID + occurred_at: datetime + revision_id: Optional[UUID] = None + revision_number: Optional[int] = None + protocol_digest: Optional[str] = None + actor: Optional[str] = None + detail: Optional[str] = None + + +class PublicationResult(BaseModel): + """Typed outcome of a publication attempt.""" + + model_config = ConfigDict(extra="forbid") + + outcome: PublicationOutcome + revision: Optional[StudyRevision] = None + errors: list[ValidationError] = Field(default_factory=list) + conflict: Optional[RevisionConflict] = None + audit: Optional[AuditRecord] = None + + +class RetirementResult(BaseModel): + """Typed outcome of retiring a published revision.""" + + model_config = ConfigDict(extra="forbid") + + retired: bool + revision: Optional[StudyRevision] = None + audit: Optional[AuditRecord] = None + message: Optional[str] = None + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +class ImmutableRevisionError(ValueError): + """Raised when a non-draft revision's content would be mutated in place. + + Publication is append-only: a material edit creates a successor revision. + """ + + +def assert_revision_mutable(revision: StudyRevision) -> StudyRevision: + """Return ``revision`` when it is an editable draft, else raise. + + Only a ``DRAFT`` revision may be edited in place. A ``PUBLISHED`` (or + ``RETIRED``) revision is immutable content; callers must publish a + successor revision (which records ``supersedes_revision_id``) instead. + """ + if revision.status != RevisionStatus.DRAFT: + raise ImmutableRevisionError( + f"revision {revision.revision_id} is {revision.status.value} and " + "its content is immutable; create a successor revision instead" + ) + return revision + + +def lineage_from_revisions( + study_id: UUID, revisions: list[StudyRevision] +) -> RevisionLineage: + """Build a :class:`RevisionLineage` from any ordering of revisions.""" + if not revisions: + return RevisionLineage(study_id=study_id) + latest = max(revisions, key=lambda revision: revision.revision_number) + return RevisionLineage( + study_id=study_id, + latest_revision_id=latest.revision_id, + latest_revision_number=latest.revision_number, + latest_digest=latest.protocol_digest, + latest_status=latest.status, + ) + + +def publish_revision( + protocol: StudyProtocolV1, + lineage: RevisionLineage, + *, + distribution_resolver: Optional[DistributionResolver] = None, + actor_is_admin: bool = False, + expected_revision_number: Optional[int] = None, + actor: Optional[str] = None, + now: Optional[datetime] = None, +) -> PublicationResult: + """Validate and publish ``protocol`` as the next revision of its study. + + When a ``distribution_resolver`` is supplied, every condition's + ``distribution_id`` is resolved and the exact pin is FROZEN into the stored + revision before the digest is computed, so an immutable revision always + records which artifact produced its data. The prior revision is never + mutated. Validation is performed before the concurrency check so an unsafe + document is always reported as ``VALIDATION_FAILED`` rather than as a + conflict; only blocking errors stop publication (warnings are advisory). + """ + timestamp = _now(now) + + frozen = ( + freeze_protocol_distributions(protocol, distribution_resolver, now=timestamp) + if distribution_resolver is not None + else protocol + ) + errors = validate_protocol( + frozen, + distribution_resolver=distribution_resolver, + actor_is_admin=actor_is_admin, + now=timestamp, + ) + blocking = blocking_errors(errors) + if blocking: + return PublicationResult( + outcome=PublicationOutcome.VALIDATION_FAILED, + errors=errors, + ) + + if ( + expected_revision_number is not None + and expected_revision_number != lineage.latest_revision_number + ): + return PublicationResult( + outcome=PublicationOutcome.CONFLICT, + conflict=RevisionConflict( + expected_revision_number=expected_revision_number, + actual_revision_number=lineage.latest_revision_number, + ), + audit=AuditRecord( + event_type="study.revision.publish_conflict", + study_id=protocol.study_id, + occurred_at=timestamp, + actor=actor, + detail=( + "expected revision " + f"{expected_revision_number}, found " + f"{lineage.latest_revision_number}" + ), + ), + ) + + digest = protocol_digest(frozen) + revision = StudyRevision( + revision_id=uuid4(), + study_id=protocol.study_id, + revision_number=lineage.latest_revision_number + 1, + status=RevisionStatus.PUBLISHED, + protocol_json=canonicalize_protocol(frozen), + protocol_digest=digest, + published_at=timestamp, + supersedes_revision_id=lineage.latest_revision_id, + created_at=timestamp, + ) + audit = AuditRecord( + event_type="study.revision.published", + study_id=protocol.study_id, + revision_id=revision.revision_id, + revision_number=revision.revision_number, + protocol_digest=digest, + actor=actor, + occurred_at=timestamp, + detail=( + f"supersedes revision {lineage.latest_revision_id}" + if lineage.latest_revision_id + else "initial publication" + ), + ) + return PublicationResult( + outcome=PublicationOutcome.PUBLISHED, + revision=revision, + audit=audit, + ) + + +def retire_revision( + revision: StudyRevision, + *, + actor: Optional[str] = None, + now: Optional[datetime] = None, +) -> RetirementResult: + """Return a retired copy of a published revision. + + The stored protocol bytes and digest are untouched; retirement is a + lifecycle status transition only, and it never changes meaning for data + already collected. The caller persists the transition via the store. + """ + timestamp = _now(now) + if revision.status == RevisionStatus.RETIRED: + return RetirementResult( + retired=False, + revision=revision, + message="revision is already retired", + ) + + retired = revision.model_copy(update={"status": RevisionStatus.RETIRED}) + audit = AuditRecord( + event_type="study.revision.retired", + study_id=revision.study_id, + revision_id=revision.revision_id, + revision_number=revision.revision_number, + protocol_digest=revision.protocol_digest, + actor=actor, + occurred_at=timestamp, + ) + return RetirementResult(retired=True, revision=retired, audit=audit) diff --git a/src/research/study/protocol/store.py b/src/research/study/protocol/store.py new file mode 100644 index 0000000..193fe16 --- /dev/null +++ b/src/research/study/protocol/store.py @@ -0,0 +1,526 @@ +"""CRUD-style persistence helpers for study protocols and revisions. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. The router is +responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from .models import StudyProtocolV1 + from .publication import AuditRecord, StudyRevision + +from database.research_schemas import ( + RECORD_KIND_STUDY_PUBLICATION, + ResearchRecord, +) +from database.db_schemas import Study as StudyRow + +from .canonical import protocol_digest +from .enums import RevisionStatus +from .join_code import generate_join_code, normalize_join_code + +#: How many times to retry a join-code allocation on the (astronomically +#: unlikely) event of a collision before failing closed. +_JOIN_CODE_ALLOCATION_ATTEMPTS = 8 + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +@dataclass(frozen=True) +class DraftView: + """A draft is an unpublished ``study_revision`` row (``status = DRAFT``).""" + + draft_id: uuid.UUID + study_id: uuid.UUID + name: str + schema_version: str + protocol_json: dict[str, Any] + created_at: Optional[datetime] + + +@dataclass(frozen=True) +class StudyView: + """Study identity, backed by the real ``public.study`` row.""" + + study_id: uuid.UUID + name: str + description: Optional[str] + owner: Optional[str] + created_by: Optional[uuid.UUID] = None + is_research: bool = False + is_active: bool = False + starts_at: Optional[datetime] = None + ends_at: Optional[datetime] = None + created_at: Optional[datetime] = None + + +def _study_row_view(row: StudyRow) -> StudyView: + return StudyView( + study_id=row.study_id, + name=row.name, + description=row.description, + owner=None, + created_by=row.created_by, + is_research=bool(row.is_research), + is_active=bool(row.is_active), + starts_at=row.starts_at, + ends_at=row.ends_at, + created_at=row.created_at, + ) + + +def _draft_view(row: StudyRevisionRow) -> DraftView: + protocol_json = row.protocol_json or {} + metadata = protocol_json.get("metadata") or {} + return DraftView( + draft_id=row.revision_id, + study_id=row.study_id, + name=str(metadata.get("name", "")), + schema_version=str(protocol_json.get("schema_version", "1")), + protocol_json=protocol_json, + created_at=row.created_at, + ) + + +def create_study( + session: Session, + *, + study_id: uuid.UUID, + name: str, + description: Optional[str] = None, + owner: Optional[str] = None, + created_by: Optional[uuid.UUID] = None, + starts_at: Optional[datetime] = None, + ends_at: Optional[datetime] = None, + is_research: bool = True, + default_config_id: Optional[int] = None, + now: Optional[datetime] = None, +) -> StudyView: + """Insert the real ``public.study`` identity row. + + Agent research studies set ``is_research`` and never fabricate a completion + ``default_config_id``. The study starts inactive; publication reserves the + owner's one live-study slot. + """ + timestamp = now or _now() + row = StudyRow( + study_id=study_id, + name=name, + description=description, + created_by=created_by, + starts_at=starts_at or timestamp, + ends_at=ends_at, + is_active=False, + default_config_id=default_config_id, + is_research=is_research, + created_at=timestamp, + ) + session.add(row) + session.commit() + session.refresh(row) + return _study_row_view(row) + + +def get_study(session: Session, study_id: uuid.UUID) -> Optional[StudyView]: + """Fetch a study identity row by id, or ``None``.""" + row = session.get(StudyRow, study_id) + return _study_row_view(row) if row is not None else None + + +def list_studies( + session: Session, owner_user_id: Optional[uuid.UUID] = None +) -> Sequence[StudyView]: + """List studies, owner-scoped when ``owner_user_id`` is given.""" + statement = select(StudyRow) + if owner_user_id is not None: + statement = statement.where(StudyRow.created_by == owner_user_id) + statement = statement.order_by(StudyRow.created_at.asc()) + return [_study_row_view(row) for row in session.execute(statement).scalars().all()] + + +def set_study_active( + session: Session, study_id: uuid.UUID, active: bool +) -> Optional[StudyRow]: + """Reserve/release a research study's live slot. + + The partial unique index ``uq_study_owner_live_research`` makes activation + DB-safe; a concurrent activation raises ``IntegrityError``, which the caller + turns into a clear conflict. + """ + row = session.get(StudyRow, study_id) + if row is None: + return None + row.is_active = active + session.commit() + session.refresh(row) + return row + + +def get_live_research_study( + session: Session, owner_user_id: uuid.UUID +) -> Optional[StudyRow]: + """Return the owner's currently live research study, or ``None``.""" + statement = select(StudyRow).where( + StudyRow.created_by == owner_user_id, + StudyRow.is_research.is_(True), + StudyRow.is_active.is_(True), + ) + return session.execute(statement).scalars().first() + + +def deactivate_expired_research_studies( + session: Session, owner_user_id: uuid.UUID, *, now: Optional[datetime] = None +) -> int: + """Flip the owner's past-end-date research studies to inactive. + + A partial unique index on ``is_active`` cannot react to time passing, so an + expired study must be actively released before a new publication reserves the + owner's slot. Only studies with a non-null, past ``ends_at`` are released. + Returns the number deactivated. + """ + timestamp = now or _now() + statement = select(StudyRow).where( + StudyRow.created_by == owner_user_id, + StudyRow.is_research.is_(True), + StudyRow.is_active.is_(True), + StudyRow.ends_at.is_not(None), + StudyRow.ends_at < timestamp, + ) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + row.is_active = False + if rows: + session.commit() + return len(rows) + + +def research_study_is_open(study: Any, now: Optional[datetime] = None) -> bool: + """Whether a research study is currently open for execution. + + ``is_active`` alone is a writer-set flag; an ended window must refuse + behaviourally even before a writer flips the flag, so the reader path checks + the schedule too. + """ + if study is None or not getattr(study, "is_research", False): + return False + if not getattr(study, "is_active", False): + return False + timestamp = now or _now() + starts_at = getattr(study, "starts_at", None) + ends_at = getattr(study, "ends_at", None) + if starts_at is not None and timestamp < starts_at: + return False + if ends_at is not None and timestamp > ends_at: + return False + return True + + +def _next_draft_number(session: Session, study_id: uuid.UUID) -> int: + """Return a draft-only revision number that never collides with publication. + + Published revisions use positive numbers; drafts use 0 or negative numbers so + publishing after a draft can never hit the ``(study_id, revision_number)`` + uniqueness constraint. + """ + statement = select(StudyRevisionRow).where(StudyRevisionRow.study_id == study_id) + numbers = [ + row.revision_number + for row in session.execute(statement).scalars().all() + if isinstance(getattr(row, "revision_number", None), int) + ] + lowest = min(numbers) if numbers else 0 + return min(lowest, 0) - 1 + + +def create_draft( + session: Session, + *, + draft_id: uuid.UUID, + study_id: uuid.UUID, + name: str, + protocol: StudyProtocolV1, + now: Optional[datetime] = None, +) -> DraftView: + """Insert an editable draft as an unpublished revision row. + + ``name`` is derived from ``protocol.metadata.name`` (the protocol document + is the draft's canonical content); ``draft_id`` is the revision id. + """ + timestamp = now or _now() + protocol_json = protocol.model_dump(mode="json") + row = StudyRevisionRow( + revision_id=draft_id, + study_id=study_id, + revision_number=_next_draft_number(session, study_id), + status=RevisionStatus.DRAFT.value, + protocol_json=protocol_json, + protocol_digest=protocol_digest(protocol), + published_at=None, + supersedes_revision_id=None, + created_at=timestamp, + ) + session.add(row) + session.commit() + session.refresh(row) + return _draft_view(row) + + +def get_draft( + session: Session, draft_id: uuid.UUID +) -> Optional[DraftView]: + """Fetch a draft revision by id, or ``None`` if it is not a draft.""" + row = session.get(StudyRevisionRow, draft_id) + if row is None or row.status != RevisionStatus.DRAFT.value: + return None + return _draft_view(row) + + +def list_drafts( + session: Session, study_id: uuid.UUID +) -> Sequence[DraftView]: + """List a study's draft revisions oldest-first.""" + statement = ( + select(StudyRevisionRow) + .where( + StudyRevisionRow.study_id == study_id, + StudyRevisionRow.status == RevisionStatus.DRAFT.value, + ) + .order_by(StudyRevisionRow.created_at.asc()) + ) + return [ + _draft_view(row) for row in session.execute(statement).scalars().all() + ] + + +def persist_revision( + session: Session, revision: StudyRevision +) -> StudyRevisionRow: + """Insert one immutable revision. + + A ``PUBLISHED`` revision additionally receives a fresh, unique join code, so + the participant onboarding handle is minted atomically with publication and + can never point at a draft. Conditions are part of ``protocol_json`` (its + canonical content), so no child rows are written. The caller owns the + transaction boundary decisions around conflicts + (``study_id``/``revision_number`` is unique). + """ + created_at = revision.created_at or _now() + published = revision.status == RevisionStatus.PUBLISHED + join_code = allocate_join_code(session) if published else None + row = StudyRevisionRow( + revision_id=revision.revision_id, + study_id=revision.study_id, + revision_number=revision.revision_number, + status=revision.status.value, + join_code=join_code, + protocol_json=revision.protocol_json, + protocol_digest=revision.protocol_digest, + published_at=revision.published_at, + supersedes_revision_id=revision.supersedes_revision_id, + created_at=created_at, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def _join_code_taken(session: Session, join_code: str) -> bool: + """Whether ``join_code`` is already allocated to a stored revision.""" + row = get_revision_by_join_code(session, join_code) + if row is None: + return False + return getattr(row, "join_code", None) == join_code + + +def allocate_join_code(session: Session) -> str: + """Return a join code not yet present in the store. + + Collisions are checked against the stored revisions (and additionally + guarded by the ``uq_study_revision_join_code`` unique constraint). The loop + is bounded so an exhausted allocator fails closed rather than spinning. + """ + for _ in range(_JOIN_CODE_ALLOCATION_ATTEMPTS): + candidate = generate_join_code() + if not _join_code_taken(session, candidate): + return candidate + raise RuntimeError("unable to allocate a unique study join code") # pragma: no cover + + +def get_revision_by_join_code( + session: Session, join_code: str +) -> Optional[StudyRevisionRow]: + """Fetch the revision a (possibly user-typed) join code resolves to. + + The code is normalized before lookup, so case/separator and Crockford + look-alike differences still resolve. An empty code never resolves. + """ + normalized = normalize_join_code(join_code) + if not normalized: + return None + statement = select(StudyRevisionRow).where( + StudyRevisionRow.join_code == normalized + ) + return session.execute(statement).scalars().first() + + +def get_latest_published_revision( + session: Session, study_id: uuid.UUID +) -> Optional[StudyRevisionRow]: + """Fetch a study's current (highest-numbered) ``PUBLISHED`` revision. + + Retired revisions are excluded: the current onboarding handle is always the + live published revision. + """ + statement = ( + select(StudyRevisionRow) + .where( + StudyRevisionRow.study_id == study_id, + StudyRevisionRow.status == RevisionStatus.PUBLISHED.value, + ) + .order_by(StudyRevisionRow.revision_number.desc()) + ) + return session.execute(statement).scalars().first() + + +def get_study_join_code( + session: Session, study_id: uuid.UUID +) -> Optional[StudyRevisionRow]: + """Fetch the current join-code-bearing revision for a study, or ``None``.""" + row = get_latest_published_revision(session, study_id) + if row is None or not getattr(row, "join_code", None): + return None + return row + + +def get_revision( + session: Session, revision_id: uuid.UUID +) -> Optional[StudyRevisionRow]: + """Fetch a revision row by primary key, or ``None``.""" + return session.get(StudyRevisionRow, revision_id) + + +def list_revisions( + session: Session, study_id: uuid.UUID +) -> Sequence[StudyRevisionRow]: + """List a study's published/retired revisions in ascending revision order. + + Draft rows share the table but are not revisions of the publication lineage, + so they are excluded here (use :func:`list_drafts` for those). + """ + statement = ( + select(StudyRevisionRow) + .where( + StudyRevisionRow.study_id == study_id, + StudyRevisionRow.status != RevisionStatus.DRAFT.value, + ) + .order_by(StudyRevisionRow.revision_number.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def retire_revision( + session: Session, revision_id: uuid.UUID, *, now: Optional[datetime] = None +) -> Optional[StudyRevisionRow]: + """Flip a revision's lifecycle status to ``RETIRED`` without touching bytes.""" + row = session.get(StudyRevisionRow, revision_id) + if row is None: + return None + row.status = RevisionStatus.RETIRED.value + session.commit() + session.refresh(row) + return row + + +def persist_audit( + session: Session, audit: AuditRecord +) -> ResearchRecord: + """Append a publication/lifecycle audit record to the generic record table.""" + row = ResearchRecord( + record_id=uuid.uuid4(), + kind=RECORD_KIND_STUDY_PUBLICATION, + scope_type="study", + scope_id=audit.study_id, + study_id=audit.study_id, + actor=audit.actor, + occurred_at=audit.occurred_at, + payload_json={ + "action": audit.event_type, + "revision_id": ( + str(audit.revision_id) if audit.revision_id is not None else None + ), + "revision_number": audit.revision_number, + "protocol_digest": audit.protocol_digest, + "detail": audit.detail, + }, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def row_to_revision(row: StudyRevisionRow) -> StudyRevision: + """Rehydrate a stored revision row into the domain model.""" + from .publication import StudyRevision as StudyRevisionModel + + status = row.status + return StudyRevisionModel( + revision_id=row.revision_id, + study_id=row.study_id, + revision_number=row.revision_number, + status=status + if isinstance(status, RevisionStatus) + else RevisionStatus(status), + protocol_json=row.protocol_json, + protocol_digest=row.protocol_digest, + published_at=row.published_at, + supersedes_revision_id=row.supersedes_revision_id, + created_at=row.created_at, + ) + + +def row_to_protocol(row: StudyRevisionRow) -> StudyProtocolV1: + """Rehydrate and validate a stored revision's protocol document.""" + from .models import StudyProtocolV1 as StudyProtocolV1Model + + return StudyProtocolV1Model.model_validate(row.protocol_json) + + +def revision_summary(row: StudyRevisionRow) -> dict[str, Any]: + """Return a compact, non-secret revision summary safe for list responses.""" + published_at = row.published_at + created_at = row.created_at + return { + "revision_id": str(row.revision_id), + "study_id": str(row.study_id), + "revision_number": row.revision_number, + "status": row.status.value if isinstance(row.status, RevisionStatus) else row.status, + "join_code": getattr(row, "join_code", None), + "protocol_digest": row.protocol_digest, + "supersedes_revision_id": ( + str(row.supersedes_revision_id) + if row.supersedes_revision_id is not None + else None + ), + "published_at": ( + published_at.isoformat() if isinstance(published_at, datetime) else None + ), + "created_at": ( + created_at.isoformat() if isinstance(created_at, datetime) else None + ), + } diff --git a/src/research/study/protocol/validation.py b/src/research/study/protocol/validation.py new file mode 100644 index 0000000..e50bd34 --- /dev/null +++ b/src/research/study/protocol/validation.py @@ -0,0 +1,990 @@ +"""Field-level, cross-reference and safety validation for study protocols. + +Validation never fails open. Every rejection is a typed :class:`ValidationError` +with a stable machine-readable +:class:`~research.study.protocol.enums.ValidationReasonCode` and a dotted ``field`` +path. Cross-validation against the agent registry happens through an injected +:class:`ReleaseResolver`, so this package never imports the registry. + +Weight normalization +-------------------- + +Condition weights are *relative* frequencies, not required to sum to one. The +deterministic normalization is ``w_i / sum(w)`` and is exposed through +:func:`normalized_condition_weights`. A protocol whose weights do not form a +normalizable positive total (missing, non-positive per condition, or a +non-positive/NaN total) is rejected with ``NON_POSITIVE_WEIGHT`` / +``EMPTY_WEIGHT_TOTAL`` rather than silently repaired. +""" + +from __future__ import annotations + +import math +import re +from datetime import datetime, timezone +from typing import Any, Mapping, Optional, Protocol, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field +from pydantic import ValidationError as PydanticValidationError + +from research.compatibility.enums import CapabilityId, CapabilityState + +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + ReleaseResolutionStatus, + RetentionAction, + ValidationReasonCode, + ValidationSeverity, +) +from .models import ( + ExplicitUnknown, + FixedSchedule, + ResolvedAgentConfig, + ResolvedDistribution, + RollingSchedule, + StudyCondition, + StudyProtocolV1, +) + +_VALID_SCHEMA_VERSIONS = frozenset({"1"}) + +#: Known distribution contracts a release pin may declare. +_DISTRIBUTION_MODES = frozenset({"PACKAGED", "BYOA_EXTERNAL"}) + +# Key substrings that identify participant/session/account data or credentials. +# ``_SAFE_KEY_NAMES`` lists legitimate, non-secret keys that merely *contain* a +# flagged substring (e.g. a token count), so they are not false positives. +_SAFE_KEY_NAMES = frozenset( + { + "max_context_tokens", + "max_tokens", + "prompt_tokens", + "completion_tokens", + "total_tokens", + # The funding owner is deliberately frozen (non-secret) so connection + # authorization runs against the study owner, not the participant. It is + # a UUID, never an email/name, and never a credential. + "funding_owner_user_id", + } +) +_IDENTIFIER_KEY_PARTS = ( + "participant", + "session_id", + "session_token", + "user_id", + "account_id", + "subject_id", +) +_CREDENTIAL_KEY_PARTS = ( + "credential", + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "private_key", + "access_key", + "client_secret", + "cookie", +) +_CREDENTIAL_VALUE_PATTERNS = ( + re.compile(r"\bsk-[A-Za-z0-9]{8,}"), + re.compile(r"\bghp_[A-Za-z0-9]{8,}"), + re.compile(r"\bAKIA[A-Z0-9]{8,}"), + re.compile(r"\bBearer\s+[A-Za-z0-9._-]{8,}", re.IGNORECASE), +) + +# Local absolute filesystem paths are never part of a portable, participant- +# independent protocol (Issue 02 section 8): they leak a machine layout and +# cannot be resolved on another host. Detection is intentionally conservative +# so URLs and relative references are not rejected. +_LOCAL_PATH_PATTERNS = ( + re.compile(r"^[A-Za-z]:[\\/]"), + re.compile( + r"^/(?:Users|home|root|Volumes|private|opt|usr|etc|var|tmp|Applications)(?:/|$)" + ), + re.compile(r"(?:^|[\s\"'(=])/(?:Users|home|root)/"), +) + +_KNOWN_CAPABILITIES = frozenset(capability.value for capability in CapabilityId) +_KNOWN_CAPABILITY_STATES = frozenset(state.value for state in CapabilityState) + + +class ProtocolValidationError(ValueError): + """Raised by imperative helpers; ``validate_protocol`` never raises for data.""" + + +class ValidationError(BaseModel): + """One typed reason a protocol cannot be published.""" + + model_config = ConfigDict(extra="forbid") + + code: ValidationReasonCode + field: str + message: str + severity: ValidationSeverity = ValidationSeverity.ERROR + + +class ReleaseResolution(BaseModel): + """Outcome of resolving one pinned agent release.""" + + model_config = ConfigDict(extra="forbid") + + status: ReleaseResolutionStatus + agent_id: str + release_id: Optional[str] = None + version: Optional[str] = None + artifact_digest: Optional[str] = None + distribution_mode: str = "PACKAGED" + message: Optional[str] = None + + +class ReleaseResolver(Protocol): + """Interface the registry (or a test double) implements for cross-checks.""" + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: # pragma: no cover - structural protocol + ... + + +class NullReleaseResolver: + """Resolver that performs no external lookup. + + A reference is treated as resolved when it is *qualified* (has a + ``release_id`` or ``version``). Digest pinning and ``latest`` are enforced + structurally by the validator, so this is the correct default when no + registry backend is wired up: it never invents a digest and never fails open + on an unqualified reference. + """ + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: + if not release_id and not version: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message="release reference names no release_id or version", + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release_id, + version=version, + ) + + +class DistributionResolution(BaseModel): + """Resolved, non-secret view of one distribution (an ``AgentProfile``). + + Produced by a :class:`DistributionResolver` that has database access to the + profile row and the release registry. The protocol validator never touches + the database; it consumes this view to emit typed reasons. ``verified`` is + DERIVED by the resolver from conformance evidence (never caller-supplied), + and ``PACKAGED`` distributions additionally expose the resolved release + identity and (platform-selected) artifact digest. ``BYOA_EXTERNAL`` + distributions expose a command/package identity and are never verified. + """ + + model_config = ConfigDict(extra="forbid") + + found: bool = False + distribution_id: Optional[UUID] = None + distribution_mode: str = "PACKAGED" + release_id: Optional[str] = None + agent_id: Optional[str] = None + version: Optional[str] = None + artifact_digest: Optional[str] = None + agent_package: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + verified: bool = False + release_status: ReleaseResolutionStatus = ReleaseResolutionStatus.NOT_FOUND + message: Optional[str] = None + # Full non-secret profile config, frozen at publication (optional so pure + # protocol tests and legacy revisions remain valid). + agent_config: Optional[ResolvedAgentConfig] = None + + def to_frozen(self, *, resolved_at: Optional[datetime] = None) -> ResolvedDistribution: + """Return the immutable pin written into a published revision.""" + return ResolvedDistribution( + distribution_id=self.distribution_id + or UUID(int=0), + distribution_mode=self.distribution_mode, + release_id=self.release_id, + agent_id=self.agent_id, + version=self.version, + agent_package=self.agent_package, + agent_command=self.agent_command, + agent_command_args=list(self.agent_command_args), + artifact_digest=self.artifact_digest, + verified=self.verified, + resolved_at=resolved_at, + agent_config=self.agent_config, + ) + + +class DistributionResolver(Protocol): + """Interface a distribution backend (or a test double) implements.""" + + def resolve(self, distribution_id: UUID) -> DistributionResolution: # pragma: no cover + ... + + +class NullDistributionResolver: + """Resolver that performs no external lookup. + + Without a wired backend a *draft* cannot be resolved, so every distribution + is reported as not found. A published document that already carries its + frozen ``resolved_distribution`` is validated from that pin directly (see + :func:`_resolve_distribution`), so this default is correct for pure protocol + tests and never fails open on an unqualified draft. + """ + + def resolve(self, distribution_id: UUID) -> DistributionResolution: + return DistributionResolution(found=False, distribution_id=distribution_id) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _iter_nodes(value: Any, path: str = "") -> Any: + """Yield ``(path, key, scalar)`` triples for every node in ``value``.""" + if isinstance(value, Mapping): + for key, item in value.items(): + child = f"{path}.{key}" if path else str(key) + yield child, str(key), item + yield from _iter_nodes(item, child) + elif isinstance(value, list): + for index, item in enumerate(value): + child = f"{path}[{index}]" + yield from _iter_nodes(item, child) + + +def _scan_document(data: Any) -> list[ValidationError]: + """Reject participant/session/account identifiers and credentials.""" + errors: list[ValidationError] = [] + seen: set[tuple[str, str]] = set() + + for path, key, value in _iter_nodes(data): + lowered = key.lower() + if lowered in _SAFE_KEY_NAMES: + continue + if any(part in lowered for part in _IDENTIFIER_KEY_PARTS): + signature = (path, "identifier") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_IDENTIFIER, + field=path, + message=( + "protocol must not contain participant/session/account " + f"identifiers (found key {key!r})" + ), + ) + ) + elif any(part in lowered for part in _CREDENTIAL_KEY_PARTS): + signature = (path, "credential") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_CREDENTIAL, + field=path, + message=( + f"protocol must not contain credentials (found key {key!r})" + ), + ) + ) + elif isinstance(value, str) and any( + pattern.search(value) for pattern in _CREDENTIAL_VALUE_PATTERNS + ): + signature = (path, "credential-value") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_CREDENTIAL, + field=path, + message="protocol value looks like a credential", + ) + ) + elif isinstance(value, str) and any( + pattern.search(value) for pattern in _LOCAL_PATH_PATTERNS + ): + signature = (path, "local-path") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_LOCAL_PATH, + field=path, + message="protocol values must not contain local absolute paths", + ) + ) + + return errors + + +def _schema_error(error: PydanticValidationError) -> ValidationError: + details = error.errors() + first = details[0] if details else {} + location = first.get("loc") or () + field = ".".join(str(part) for part in location) + return ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field=field, + message=str(first.get("msg", "protocol document failed schema validation")), + ) + + +def _is_latest(value: Optional[str]) -> bool: + return bool(value) and str(value).strip().lower() == "latest" + + +def _check_conditions(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + conditions = protocol.conditions + + if not conditions: + errors.append( + ValidationError( + code=ValidationReasonCode.NO_CONDITIONS, + field="conditions", + message="a study protocol must declare at least one condition", + ) + ) + return errors + + seen: set[str] = set() + for index, condition in enumerate(conditions): + field = f"conditions[{index}]" + if condition.condition_id in seen: + errors.append( + ValidationError( + code=ValidationReasonCode.DUPLICATE_CONDITION_ID, + field=f"{field}.condition_id", + message=f"condition_id {condition.condition_id!r} is duplicated", + ) + ) + seen.add(condition.condition_id) + + weight = condition.weight + if weight is None or not math.isfinite(weight) or weight <= 0: + errors.append( + ValidationError( + code=ValidationReasonCode.NON_POSITIVE_WEIGHT, + field=f"{field}.weight", + message="condition weight must be a finite positive number", + ) + ) + + finite_weights = [ + condition.weight + for condition in conditions + if condition.weight is not None and math.isfinite(condition.weight) + ] + if not finite_weights or sum(finite_weights) <= 0: + errors.append( + ValidationError( + code=ValidationReasonCode.EMPTY_WEIGHT_TOTAL, + field="conditions", + message="condition weights must sum to a positive total", + ) + ) + + return errors + + +def _check_schedule(protocol: StudyProtocolV1, now: datetime) -> list[ValidationError]: + schedule = protocol.schedule + if isinstance(schedule, FixedSchedule): + if schedule.end_at is not None and schedule.end_at <= schedule.start_at: + return [ + ValidationError( + code=ValidationReasonCode.FIXED_SCHEDULE_INVALID, + field="schedule.end_at", + message="fixed schedule end_at must be after start_at", + ) + ] + if schedule.end_at is not None and schedule.end_at < now: + return [ + ValidationError( + code=ValidationReasonCode.FIXED_SCHEDULE_EXPIRED, + field="schedule.end_at", + message="fixed schedule already ended", + ) + ] + return [] + + if isinstance(schedule, RollingSchedule): + if schedule.duration_seconds is None or schedule.duration_seconds <= 0: + return [ + ValidationError( + code=ValidationReasonCode.ROLLING_DURATION_INVALID, + field="schedule.duration_seconds", + message="rolling schedule duration_seconds must be present and > 0", + ) + ] + return [] + + return [ + ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field="schedule.kind", + message="schedule must be FIXED or ROLLING", + ) + ] + + +def _check_assignment(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + assignment = protocol.assignment + + if assignment.unit != AssignmentUnit.ENROLLMENT: + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_UNIT_NOT_ENROLLMENT, + field="assignment.unit", + message="issue 02 only supports enrollment-level assignment", + ) + ) + + known_strategies = {strategy.value for strategy in AssignmentStrategy} + if assignment.strategy not in known_strategies: + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_STRATEGY_UNKNOWN, + field="assignment.strategy", + message=f"unknown assignment strategy {assignment.strategy!r}", + ) + ) + elif ( + assignment.strategy == AssignmentStrategy.STRATIFIED.value + and not assignment.stratification + ): + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_STRATIFICATION_MISSING, + field="assignment.stratification", + message="stratified assignment requires stratification keys", + ) + ) + + return errors + + +def _resolve_distribution( + condition: StudyCondition, + resolver: Optional[DistributionResolver], +) -> DistributionResolution: + """Resolve one condition's distribution to its non-secret view. + + A wired :class:`DistributionResolver` is authoritative. Without one, a + published revision's frozen ``resolved_distribution`` is trusted + structurally; a draft that only names an id is reported as not found. + """ + if resolver is not None: + return resolver.resolve(condition.distribution_id) + frozen = condition.resolved_distribution + if frozen is None: + return DistributionResolution( + found=False, distribution_id=condition.distribution_id + ) + return DistributionResolution( + found=True, + distribution_id=frozen.distribution_id, + distribution_mode=frozen.distribution_mode, + release_id=frozen.release_id, + agent_id=frozen.agent_id, + version=frozen.version, + artifact_digest=frozen.artifact_digest, + agent_package=frozen.agent_package, + agent_command=frozen.agent_command, + agent_command_args=list(frozen.agent_command_args), + verified=frozen.verified, + release_status=( + ReleaseResolutionStatus.RESOLVED + if frozen.verified + else ReleaseResolutionStatus.UNQUALIFIED + ), + ) + + +def _unverified_error( + field: str, + *, + distribution: DistributionResolution, + actor_is_admin: bool, +) -> ValidationError: + """One typed reason a distribution is not verified. + + A non-administrator researcher may only author a condition from a VERIFIED + distribution, so this is a hard ``ERROR`` for them. An administrator may + deliberately select an unverified distribution: it is flagged as a + ``WARNING`` that publish surfaces but does not block. + """ + mode = str(distribution.distribution_mode or "PACKAGED").strip().upper() + if mode == "BYOA_EXTERNAL": + message = ( + "a BYOA_EXTERNAL distribution is always unverified: it has no " + "artifact to bind conformance evidence to" + ) + else: + message = ( + "the distribution's release is not qualified by a passing " + "conformance receipt (unverified)" + ) + return ValidationError( + code=ValidationReasonCode.DISTRIBUTION_UNVERIFIED, + field=field, + message=message, + severity=( + ValidationSeverity.WARNING + if actor_is_admin + else ValidationSeverity.ERROR + ), + ) + + +def _check_distributions( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + actor_is_admin: bool, +) -> list[ValidationError]: + errors: list[ValidationError] = [] + + for index, condition in enumerate(protocol.conditions): + field = f"conditions[{index}].distribution_id" + frozen = condition.resolved_distribution + + distribution = _resolve_distribution(condition, resolver) + if not distribution.found: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_PROFILE_NOT_FOUND, + field=field, + message=( + f"distribution {condition.distribution_id} does not exist" + ), + ) + ) + continue + + mode = str(distribution.distribution_mode or "PACKAGED").strip().upper() + if mode not in _DISTRIBUTION_MODES: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_MODE_UNKNOWN, + field=f"{field}.distribution_mode", + message=( + "distribution_mode must be one of " + + ", ".join(sorted(_DISTRIBUTION_MODES)) + ), + ) + ) + continue + + if _is_latest(distribution.release_id) or _is_latest(distribution.version): + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_LATEST, + field=field, + message="a distribution must pin a release, not 'latest'", + ) + ) + + if mode == "BYOA_EXTERNAL": + # A participant-installed agent is resolved from a command/package + # identity, never from an artifact digest. + if not ( + distribution.release_id + or distribution.agent_package + or distribution.agent_command + or distribution.agent_id + ): + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_IDENTITY_REQUIRED, + field=field, + message=( + "a BYOA_EXTERNAL distribution must declare an " + "agent_package or agent_command identity" + ), + ) + ) + # No conformance evidence can be bound to a participant-installed + # agent, so a BYOA distribution is always unverified. + errors.append( + _unverified_error( + field, distribution=distribution, actor_is_admin=actor_is_admin + ) + ) + continue + + # PACKAGED: the distribution MUST pin a resolvable release id. + if not distribution.release_id: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_UNPINNED, + field=field, + message="a PACKAGED distribution must pin a release_id", + ) + ) + continue + + # A PACKAGED pin must never embed a mutable executable command. + if frozen is not None and frozen.agent_command: + errors.append( + ValidationError( + code=ValidationReasonCode.MUTABLE_AGENT_COMMAND, + field=f"{field}.resolved_distribution.agent_command", + message=( + "a PACKAGED distribution must not embed an executable " + "command" + ), + ) + ) + + if distribution.release_status == ReleaseResolutionStatus.NOT_FOUND: + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_UNRESOLVED, + field=field, + message=( + f"release {distribution.release_id!r} could not be resolved" + ), + ) + ) + elif distribution.release_status == ReleaseResolutionStatus.WITHDRAWN: + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_WITHDRAWN, + field=field, + message=( + f"release {distribution.release_id!r} has been withdrawn" + ), + ) + ) + elif not distribution.verified: + # The release exists but carries no passing conformance receipt. + errors.append( + _unverified_error( + field, distribution=distribution, actor_is_admin=actor_is_admin + ) + ) + elif not distribution.artifact_digest: + # A verified PACKAGED distribution must still resolve to a concrete + # digest-pinned artifact; without one there is nothing to launch. + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_UNPINNED, + field=field, + message=( + f"release {distribution.release_id!r} publishes no " + "digest-pinned artifact" + ), + ) + ) + + # A frozen digest that disagrees with the registry is a hard block: the + # immutable revision no longer describes the artifact it claims to. + if ( + frozen is not None + and frozen.artifact_digest + and distribution.artifact_digest + and frozen.artifact_digest != distribution.artifact_digest + ): + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_DIGEST_MISMATCH, + field=f"{field}.resolved_distribution.artifact_digest", + message="frozen artifact digest does not match the registry", + ) + ) + + return errors + + +def freeze_protocol_distributions( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + *, + now: Optional[datetime] = None, +) -> StudyProtocolV1: + """Return a copy of ``protocol`` with every condition's pin frozen. + + The resolved pin (release identity, artifact digest, distribution mode and + the derived ``verified`` flag) is written into ``resolved_distribution`` so + an immutable revision always records exactly which artifact produced its + data. A condition whose distribution cannot be resolved keeps ``None`` and + is rejected by validation rather than silently frozen. + """ + timestamp = _now(now) + frozen_conditions: list[StudyCondition] = [] + for condition in protocol.conditions: + distribution = _resolve_distribution(condition, resolver) + frozen_conditions.append( + condition.model_copy( + update={ + "resolved_distribution": ( + distribution.to_frozen(resolved_at=timestamp) + if distribution.found + else None + ) + } + ) + ) + return protocol.model_copy(update={"conditions": frozen_conditions}) + + +def _check_environment(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + environment = protocol.environment_requirements + + if not environment.expected_protocol_version: + errors.append( + ValidationError( + code=ValidationReasonCode.ENVIRONMENT_PROTOCOL_VERSION_MISSING, + field="environment_requirements.expected_protocol_version", + message="an expected ACP protocol version is required", + ) + ) + + for index, required in enumerate(environment.required_capabilities): + capability = required.capability + field = f"environment_requirements.required_capabilities[{index}]" + if capability not in _KNOWN_CAPABILITIES: + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_REQUIRED_CAPABILITY, + field=field, + severity=ValidationSeverity.NEEDS_REVIEW, + message=( + f"required capability {capability!r} is not in the known " + "capability vocabulary" + ), + ) + ) + elif required.require_state not in _KNOWN_CAPABILITY_STATES: + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_REQUIRED_CAPABILITY, + field=f"{field}.require_state", + severity=ValidationSeverity.NEEDS_REVIEW, + message=( + f"required state {required.require_state!r} is not a known " + "capability state" + ), + ) + ) + + if isinstance(environment.host_kind, ExplicitUnknown): + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_ENVIRONMENT_REQUIREMENT, + field="environment_requirements.host_kind", + severity=ValidationSeverity.NEEDS_REVIEW, + message="host_kind is explicitly unknown and must be resolved", + ) + ) + + return errors + + +def _check_retention(protocol: StudyProtocolV1) -> list[ValidationError]: + """Reject retention policies that are not supported to publish. + + The supported policy retains uploaded data according to consent. A + participant-selected ``DELETE_ALL`` is not authorable: advertising a + deletion policy that is not the executed one is misleading. + """ + if protocol.privacy_policy.retention_action == RetentionAction.DELETE_ALL: + return [ + ValidationError( + code=ValidationReasonCode.RETENTION_UNSUPPORTED, + field="privacy_policy.retention_action", + message=( + "DELETE_ALL is not a supported published retention policy; " + "retain uploaded data according to consent" + ), + ) + ] + return [] + + +def _check_completion(protocol: StudyProtocolV1) -> list[ValidationError]: + completion = protocol.completion + if completion.policy == CompletionPolicyKind.TARGET_CAPACITY and ( + completion.target_enrollments is None or completion.target_enrollments <= 0 + ): + return [ + ValidationError( + code=ValidationReasonCode.COMPLETION_TARGET_INVALID, + field="completion.target_enrollments", + message="TARGET_CAPACITY completion requires a positive target", + ) + ] + return [] + + +def _check_protocol( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + now: datetime, + *, + actor_is_admin: bool = False, +) -> list[ValidationError]: + errors: list[ValidationError] = [] + + if protocol.schema_version not in _VALID_SCHEMA_VERSIONS: + errors.append( + ValidationError( + code=ValidationReasonCode.UNSUPPORTED_SCHEMA_VERSION, + field="schema_version", + message=( + f"unsupported schema_version {protocol.schema_version!r}; " + f"expected one of {sorted(_VALID_SCHEMA_VERSIONS)}" + ), + ) + ) + + errors.extend(_check_conditions(protocol)) + errors.extend(_check_schedule(protocol, now)) + errors.extend(_check_assignment(protocol)) + errors.extend(_check_retention(protocol)) + errors.extend(_check_environment(protocol)) + errors.extend(_check_completion(protocol)) + errors.extend(_check_distributions(protocol, resolver, actor_is_admin)) + return errors + + +def validate_protocol( + document: Union[StudyProtocolV1, Mapping[str, Any]], + *, + distribution_resolver: Optional[DistributionResolver] = None, + actor_is_admin: bool = False, + now: Optional[datetime] = None, +) -> list[ValidationError]: + """Return every reason ``document`` cannot be published. + + An empty list means the document is publishable. A raw mapping is scanned + for forbidden identifiers/credentials *before* schema parsing so leakage is + always reported with a typed reason even when the extra keys also violate + the schema. + + ``distribution_resolver`` resolves each condition's ``distribution_id`` to + its release/verification view. ``actor_is_admin`` selects the severity of a + ``DISTRIBUTION_UNVERIFIED`` reason: a hard error for a researcher, a warning + for an administrator. + """ + if isinstance(document, StudyProtocolV1): + protocol: Optional[StudyProtocolV1] = document + raw: Any = document.model_dump(mode="json") + elif isinstance(document, Mapping): + protocol = None + raw = dict(document) + else: + return [ + ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field="", + message="protocol must be a StudyProtocolV1 or a mapping", + ) + ] + + errors = _scan_document(raw) + + if protocol is None: + try: + protocol = StudyProtocolV1.model_validate(raw) + except PydanticValidationError as error: + errors.append(_schema_error(error)) + return errors + + errors.extend( + _check_protocol( + protocol, + distribution_resolver, + _now(now), + actor_is_admin=actor_is_admin, + ) + ) + return errors + + +def scan_document_safety( + document: Union[StudyProtocolV1, Mapping[str, Any]] +) -> list[ValidationError]: + """Return only the forbidden-identifier/credential findings for a document. + + Used by draft creation, which must never persist a document that leaks a + participant/session/account identifier or a credential even though the draft + is not yet publishable. + """ + if isinstance(document, StudyProtocolV1): + raw: Any = document.model_dump(mode="json") + elif isinstance(document, Mapping): + raw = document + else: + return [] + return _scan_document(raw) + + +def blocking_errors(errors: list[ValidationError]) -> list[ValidationError]: + """The subset of ``errors`` that must block publication.""" + return [ + error for error in errors if error.severity != ValidationSeverity.WARNING + ] + + +def warnings(errors: list[ValidationError]) -> list[ValidationError]: + """The informational, non-blocking subset of ``errors``.""" + return [error for error in errors if error.severity == ValidationSeverity.WARNING] + + +def is_publishable(errors: list[ValidationError]) -> bool: + """Whether a validation result permits publication. + + ``ERROR`` and ``NEEDS_REVIEW`` both block publication (the latter asks for + human review rather than an automatic fix); ``WARNING`` is informational and + does not block. + """ + return not blocking_errors(errors) + + +def normalized_condition_weights(protocol: StudyProtocolV1) -> dict[str, float]: + """Deterministic ``w_i / sum(w)`` map for a valid protocol. + + Raises :class:`ProtocolValidationError` when the weights do not form a + normalizable positive total, so a caller can never divide by zero or + silently drop a condition. + """ + weights = { + condition.condition_id: condition.weight for condition in protocol.conditions + } + total = sum(weight for weight in weights.values() if weight is not None) + if not weights or not math.isfinite(total) or total <= 0: + raise ProtocolValidationError("condition weights do not sum to a positive total") + return { + condition_id: weight / total + for condition_id, weight in weights.items() + } diff --git a/src/research/telemetry/__init__.py b/src/research/telemetry/__init__.py new file mode 100644 index 0000000..c14ff58 --- /dev/null +++ b/src/research/telemetry/__init__.py @@ -0,0 +1,73 @@ +"""Canonical telemetry schema and privacy filter (Issue 06). + +One source-neutral event vocabulary that preserves provenance and enforces data +minimization before durable storage. + +Public surface: + +* :mod:`research.telemetry.enums` - canonical event types, sources, field + classes, policy actions, coverage states, canonical fidelity and lifecycle. +* :mod:`research.telemetry.models` - the immutable ``CanonicalEventV1`` + envelope and its sub-objects. +* :mod:`research.telemetry.builder` - event construction with a fresh id and a + strictly-increasing per-emitter sequence. +* :mod:`research.telemetry.validation` - typed envelope validation. +* :mod:`research.telemetry.privacy` - field classification and the redaction + engine that runs before any spool/log/retry/export. +* :mod:`research.telemetry.normalization` - ACP/IDE/legacy source normalizers + and the optional adapter enrichment interface. +* :mod:`research.telemetry.schema` - the language-neutral JSON Schema. + +The package never imports ``App``, FastAPI, or SQLAlchemy; it is the shared +contract that Issues 07-12 consume. +""" + +from .builder import EventBuilder, SequenceAllocator, build_event +from .enums import ( + CanonicalEventType, + CanonicalFidelity, + CoverageState, + EventSource, + FieldClass, + LifecycleState, + PolicyAction, +) +from .models import ( + CanonicalEventV1, + Correlations, + Coverage, + EventMetrics, + PrivacySummary, + Provenance, +) +from .privacy import PrivacyPolicy, filter_event, filter_payload +from .validation import ( + TelemetryValidationCode, + TelemetryValidationError, + validate_canonical_event, +) + +__all__ = [ + "CanonicalEventType", + "CanonicalEventV1", + "CanonicalFidelity", + "Correlations", + "Coverage", + "CoverageState", + "EventBuilder", + "EventMetrics", + "EventSource", + "FieldClass", + "LifecycleState", + "PolicyAction", + "PrivacyPolicy", + "PrivacySummary", + "Provenance", + "SequenceAllocator", + "TelemetryValidationCode", + "TelemetryValidationError", + "build_event", + "filter_event", + "filter_payload", + "validate_canonical_event", +] diff --git a/src/research/telemetry/adapters.py b/src/research/telemetry/adapters.py new file mode 100644 index 0000000..4d01fb5 --- /dev/null +++ b/src/research/telemetry/adapters.py @@ -0,0 +1,153 @@ +"""Server-authorized adapters feeding legacy producers into the canonical stream. + +The inference relay, the self-reporting runtime and the OTel span endpoint +observe agent activity that used to be written to the legacy ``agent_event`` +table. For a *research-bound* task (an explicit ``research_session_id`` / +``enrollment_id`` / ``study_revision_id`` on ``agent_task``) those observations +are now built as canonical events and persisted through the one ingestion writer +(:func:`ingest_events_for_context`), so the dashboards read the canonical event +authority and there is no parallel canonical writer. + +A task with no research binding keeps the legacy operational path: ``agent_event`` +is retained for genuinely non-research operational data and this adapter returns +``False`` so the caller performs its legacy write. Nothing is fabricated: a fact +with no exact canonical equivalent is preserved as an explicit unknown (with its +original kind in ``payload.legacy_kind`` and provenance), never reclassified, and +a missing measurement is never turned into zero. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional, Sequence + +from research.participants import identity as identity_store +from research.runtime.sessions import store as session_store +from research.telemetry.builder import EventBuilder +from research.telemetry.enums import CanonicalEventType, EventSource +from research.telemetry.ingestion.models import IngestionContext +from research.telemetry.ingestion.service import ingest_events_for_context +from research.telemetry.ingestion.store import SqlAlchemyIngestionStore +from research.telemetry.models import Correlations, Coverage, EventMetrics + +__all__ = [ + "LegacyFact", + "RELAY_NORMALIZER_VERSION", + "record_legacy_facts", + "research_bound", +] + +RELAY_NORMALIZER_VERSION = "relay-adapter-v1" + +#: Legacy producer kind -> canonical event type. A kind with no exact canonical +#: equivalent is preserved as ``unknown_source_event`` (the builder records the +#: original kind), never reclassified into a nearby concept. +_KIND_TO_TYPE: dict[str, CanonicalEventType] = { + "model_call": CanonicalEventType.AGENT_MESSAGE_COMPLETED, + "tool_call": CanonicalEventType.TOOL_COMPLETED, + "tool_failed": CanonicalEventType.TOOL_FAILED, + "observation": CanonicalEventType.UNKNOWN_SOURCE_EVENT, +} + + +@dataclass(frozen=True) +class LegacyFact: + """One already-sanitized legacy observation, ready to canonicalize. + + ``payload`` must contain only structural metadata (no prompt/source/output/ + diff/reasoning content): content stays behind the legacy consent gate. + """ + + kind: str + occurred_at: datetime + payload: dict[str, Any] = field(default_factory=dict) + metrics: EventMetrics = field(default_factory=EventMetrics) + coverage: Coverage = field(default_factory=Coverage) + correlations: Correlations = field(default_factory=Correlations) + source_event_id: Optional[str] = None + emitter_id: str = "relay" + + +def research_bound(task: Any) -> bool: + """Whether a task carries the explicit phase-05 research binding.""" + return getattr(task, "research_session_id", None) is not None + + +def build_legacy_events(task: Any, facts: Sequence[LegacyFact]) -> list: + """Build canonical events for a research-bound task, in order.""" + builder = EventBuilder() + sequence = int(getattr(task, "next_event_index", 0) or 0) + events = [] + for fact in facts: + sequence += 1 + event_type = _KIND_TO_TYPE.get( + fact.kind, CanonicalEventType.UNKNOWN_SOURCE_EVENT + ) + events.append( + builder.build( + emitter_id=fact.emitter_id, + event_type=event_type, + source=EventSource.RELAY, + occurred_at=fact.occurred_at, + normalizer_version=RELAY_NORMALIZER_VERSION, + payload={**fact.payload, "legacy_kind": fact.kind}, + metrics=fact.metrics, + coverage=fact.coverage, + correlations=fact.correlations, + source_event_id=fact.source_event_id, + study_id=getattr(task, "study_id", None), + revision_id=getattr(task, "study_revision_id", None), + enrollment_id=getattr(task, "enrollment_id", None), + research_session_id=getattr(task, "research_session_id", None), + agent_run_id=getattr(task, "external_run_id", None), + emitter_sequence=sequence, + ) + ) + return events + + +def record_legacy_facts(db: Any, *, task: Any, facts: Sequence[LegacyFact]) -> bool: + """Persist legacy facts through the canonical ingestion writer. + + Returns ``True`` when the facts were canonicalized (the caller must **not** + also write ``agent_event``). Returns ``False`` when the task has no research + binding — or its binding rows are unavailable — so the caller keeps the + legacy operational write. Never raises: a canonical-write failure returns + ``False`` so the caller's legacy path still records the observation. + """ + if task is None or not facts or not research_bound(task): + return False + try: + enrollment_row = ( + identity_store.get_enrollment(db, task.enrollment_id) + if getattr(task, "enrollment_id", None) is not None + else None + ) + session_row = ( + session_store.get_session(db, task.research_session_id) + if getattr(task, "research_session_id", None) is not None + else None + ) + if enrollment_row is None or session_row is None: + return False + enrollment = identity_store.row_to_enrollment(enrollment_row) + session = session_store.row_to_session(session_row) + context = IngestionContext( + study_id=task.study_id, + enrollment_id=task.enrollment_id, + study_revision_id=task.study_revision_id, + research_session_id=task.research_session_id, + revocation_epoch=enrollment.revocation_epoch, + ) + events = build_legacy_events(task, facts) + ingest_events_for_context( + context=context, + enrollment=enrollment, + session=session, + events=events, + store=SqlAlchemyIngestionStore(db), + ) + return True + except Exception: # noqa: BLE001 - the caller keeps the legacy fallback + return False diff --git a/src/research/telemetry/builder.py b/src/research/telemetry/builder.py new file mode 100644 index 0000000..4cf4560 --- /dev/null +++ b/src/research/telemetry/builder.py @@ -0,0 +1,184 @@ +"""Canonical event construction and per-emitter sequencing. + +The builder owns two things the source mappers must not invent: + +* a fresh, immutable ``event_id``; +* a strictly-increasing ``emitter_sequence`` **per emitter** (independent + emitters each start their own sequence; there is no fake global counter). + +It never derives cross-process latency: ``latency_ms`` is only copied from an +explicit caller-provided metric. It also does not redact; callers must pass the +result through :func:`research.telemetry.privacy.engine.filter_event` before the +event reaches a spool, log, retry queue, or export. +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any, Mapping, Optional + +from .enums import ( + CANONICAL_SCHEMA_VERSION, + CanonicalEventType, + CanonicalFidelity, + CoverageState, + EventSource, + LifecycleState, +) +from .models import ( + CanonicalEventV1, + Correlations, + Coverage, + EventMetrics, + PrivacySummary, + Provenance, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + +class SequenceAllocator: + """Allocates strictly-increasing sequences independently per emitter.""" + + def __init__(self, start: int = 1) -> None: + self._start = start + self._last: dict[str, int] = {} + + def next(self, emitter_id: str) -> int: + """Return the next sequence for ``emitter_id`` (starting at ``start``).""" + value = self._last.get(emitter_id, self._start - 1) + 1 + self._last[emitter_id] = value + return value + + def current(self, emitter_id: str) -> int: + """Return the last allocated sequence for ``emitter_id`` (0 if none).""" + return self._last.get(emitter_id, self._start - 1) + + +def canonical_event_type(raw: Any) -> tuple[str, Optional[str]]: + """Return ``(canonical_type, unknown_original)`` for a raw event type.""" + if isinstance(raw, CanonicalEventType): + return raw.value, None + text = str(raw) + try: + return CanonicalEventType(text).value, None + except ValueError: + return CanonicalEventType.UNKNOWN_SOURCE_EVENT.value, text + + +def canonical_source(raw: Any) -> tuple[str, Optional[str]]: + """Return ``(canonical_source, unknown_original)`` for a raw source.""" + if isinstance(raw, EventSource): + return raw.value, None + text = str(raw) + try: + return EventSource(text.strip().lower()).value, None + except ValueError: + return text, text + + +def canonical_lifecycle(raw: Any) -> tuple[Optional[str], Optional[str]]: + """Return ``(canonical_lifecycle, unknown_original)`` for a raw state.""" + if raw is None: + return None, None + if isinstance(raw, LifecycleState): + return raw.value, None + text = str(raw) + try: + return LifecycleState(text.strip().lower()).value, None + except ValueError: + return text, text + + +class EventBuilder: + """Builds immutable :class:`CanonicalEventV1` envelopes.""" + + def __init__(self, allocator: Optional[SequenceAllocator] = None) -> None: + self.allocator = allocator or SequenceAllocator() + + def build( + self, + *, + emitter_id: str, + event_type: Any, + source: Any, + occurred_at: datetime, + normalizer_version: str, + payload: Optional[Mapping[str, Any]] = None, + fidelity: CanonicalFidelity = CanonicalFidelity.EXACT, + adapter_version: Optional[str] = None, + source_event_id: Optional[str] = None, + evidence_digest: Optional[str] = None, + metrics: Optional[EventMetrics] = None, + coverage: Optional[Coverage] = None, + privacy: Optional[PrivacySummary] = None, + lifecycle_state: Any = None, + correlations: Optional[Correlations] = None, + monotonic_ns: Optional[int] = None, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + research_session_id: Optional[UUID] = None, + agent_run_id: Optional[str] = None, + event_id: Optional[UUID] = None, + emitter_sequence: Optional[int] = None, + ) -> CanonicalEventV1: + """Build one canonical event with a fresh id and allocated sequence.""" + canonical_type, unknown_event_type = canonical_event_type(event_type) + canonical_source_value, unknown_source = canonical_source(source) + lifecycle, unknown_lifecycle = canonical_lifecycle(lifecycle_state) + + resolved_coverage = coverage or Coverage() + if unknown_event_type or unknown_source or unknown_lifecycle: + resolved_coverage = Coverage( + state=CoverageState.NEEDS_REVIEW, + reason="unrecognized input preserved for review", + capability=resolved_coverage.capability, + ) + + return CanonicalEventV1( + event_id=event_id or uuid.uuid4(), + schema_version=CANONICAL_SCHEMA_VERSION, + event_type=canonical_type, + source=canonical_source_value, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + agent_run_id=agent_run_id, + occurred_at=occurred_at, + monotonic_ns=monotonic_ns, + emitter_id=emitter_id, + emitter_sequence=( + emitter_sequence + if emitter_sequence is not None + else self.allocator.next(emitter_id) + ), + correlations=correlations or Correlations(), + lifecycle_state=lifecycle, + payload=dict(payload or {}), + metrics=metrics or EventMetrics(), + privacy=privacy or PrivacySummary(), + provenance=Provenance( + source=canonical_source_value, + source_event_id=source_event_id, + normalizer_version=normalizer_version, + adapter_version=adapter_version, + fidelity=fidelity, + evidence_digest=evidence_digest, + ), + coverage=resolved_coverage, + unknown_event_type=unknown_event_type, + unknown_source=unknown_source, + unknown_lifecycle_state=unknown_lifecycle, + ) + + +_DEFAULT_ALLOCATOR = SequenceAllocator() + + +def build_event(**kwargs: Any) -> CanonicalEventV1: + """Build an event using the process-wide default sequence allocator.""" + return EventBuilder(_DEFAULT_ALLOCATOR).build(**kwargs) diff --git a/src/research/telemetry/enums.py b/src/research/telemetry/enums.py new file mode 100644 index 0000000..1fc8f4a --- /dev/null +++ b/src/research/telemetry/enums.py @@ -0,0 +1,146 @@ +"""Closed vocabularies for the canonical telemetry schema (Issue 06). + +Every member must have a real producer (the IDE collector, the shared ACP/proxy +normalizer, or the server); a member with no producer is removed rather than +kept as an unreachable reserved token, because the vocabulary is the persisted/ +exported contract. Otherwise members are additive only: renaming a produced +member is a breaking change to already-stored events. + +Fidelity note: :class:`CanonicalFidelity` uses the **lowercase** telemetry-spec +values (``exact`` / ``normalized`` / ``inferred``). It is deliberately distinct +from the **uppercase** receipt fidelity in +:class:`research.compatibility.enums.Fidelity` (``EXACT`` / ``NORMALIZED`` / +``INFERRED``); the two must not be conflated. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "CANONICAL_SCHEMA_VERSION", + "CanonicalEventType", + "CanonicalFidelity", + "CoverageState", + "EventSource", + "FieldClass", + "LifecycleState", + "PolicyAction", +] + +#: The canonical telemetry envelope schema version. It is required on every +#: event and is never defaulted on ingestion. +CANONICAL_SCHEMA_VERSION = "1" + + +class CanonicalEventType(str, Enum): + """Canonical, vendor-independent event concepts. + + These are canonical concepts, not ACP method names: an ACP notification is + *normalized into* one of these, and an unknown construct becomes + :attr:`UNKNOWN_SOURCE_EVENT` rather than a new vendor-named type. + + Members are additive only. A member with no producer anywhere in the + platform (plugin IDE collector, shared ACP/proxy normalizer, server) is not + part of the vocabulary: it was removed rather than left as an unreachable + reserved token. + """ + + INTERACTION_STARTED = "interaction.started" + INTERACTION_COMPLETED = "interaction.completed" + + AGENT_MESSAGE_STARTED = "agent.message.started" + AGENT_MESSAGE_COMPLETED = "agent.message.completed" + + TOOL_CREATED = "tool.created" + TOOL_STARTED = "tool.started" + TOOL_COMPLETED = "tool.completed" + TOOL_FAILED = "tool.failed" + + PERMISSION_REQUESTED = "permission.requested" + PERMISSION_DECIDED = "permission.decided" + + PLAN_UPDATED = "plan.updated" + USAGE_UPDATED = "usage.updated" + + IDE_DOCUMENT_CHANGED = "ide.document.changed" + IDE_FILE_OPENED = "ide.file.opened" + IDE_FILE_SAVED = "ide.file.saved" + IDE_FILE_CLOSED = "ide.file.closed" + IDE_RUN_EXECUTED = "ide.run.executed" + + SYSTEM_AGENT_CRASHED = "system.agent.crashed" + SYSTEM_PROXY_ERROR = "system.proxy.error" + + AGENT_ERROR = "agent.error" + + UNKNOWN_SOURCE_EVENT = "unknown_source_event" + + +class EventSource(str, Enum): + """Which observation channel emitted the source event (lowercase values).""" + + ACP = "acp" + IDE = "ide" + LEGACY = "legacy" + # The backend inference relay observes provider model calls directly; it is + # a distinct observation channel from the ACP proxy and the IDE collector, + # so source ownership can prevent double counting the same action. + RELAY = "relay" + + +class FieldClass(str, Enum): + """Data classification used to decide whether a field may persist.""" + + SYSTEM = "SYSTEM" + BEHAVIORAL = "BEHAVIORAL" + CODE_METADATA = "CODE_METADATA" + CONTENT = "CONTENT" + SECRET = "SECRET" + + +class PolicyAction(str, Enum): + """Terminal privacy decision for one field (or the whole event).""" + + ALLOW = "ALLOW" + REDACT = "REDACT" + HASH = "HASH" + DROP = "DROP" + BLOCK = "BLOCK" + + +class CoverageState(str, Enum): + """Whether a measurement/observation is present, absent, or disputed. + + ``UNAVAILABLE`` means "the source does not expose it" and ``UNKNOWN`` means + "not observed"; neither is zero or false. + """ + + AVAILABLE = "AVAILABLE" + UNAVAILABLE = "UNAVAILABLE" + UNKNOWN = "UNKNOWN" + PARTIAL = "PARTIAL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +class CanonicalFidelity(str, Enum): + """How faithfully a canonical field represents its source observation. + + Lowercase by canonical-spec convention; distinct from the uppercase receipt + fidelity vocabulary in Issue 01. + """ + + EXACT = "exact" + NORMALIZED = "normalized" + INFERRED = "inferred" + + +class LifecycleState(str, Enum): + """Lifecycle phase of a lifecycled canonical event.""" + + PENDING = "pending" + STARTED = "started" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + UNKNOWN = "unknown" diff --git a/src/research/telemetry/ingestion/__init__.py b/src/research/telemetry/ingestion/__init__.py new file mode 100644 index 0000000..1f0f0e2 --- /dev/null +++ b/src/research/telemetry/ingestion/__init__.py @@ -0,0 +1,66 @@ +"""Idempotent canonical telemetry ingestion (Issue 09). + +Public surface: + +* :mod:`research.telemetry.ingestion.enums` - dispositions and typed ingest reason codes. +* :mod:`research.telemetry.ingestion.models` - ``TelemetryBatchRequestV1`` / ``TelemetryBatchAckV1``, + event records, receipts and the :class:`~research.telemetry.ingestion.models.IngestionStore` + protocol. +* :mod:`research.telemetry.ingestion.validation` - per-event schema/context/sensitive scan. +* :mod:`research.telemetry.ingestion.service` - :func:`ingest_batch`, the deterministic + authorize-classify-persist service. +* :mod:`research.telemetry.ingestion.store` - SQLAlchemy persistence and the in-memory + :class:`~research.telemetry.ingestion.store.FakeIngestionStore` test double. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ( + PERMANENT_REASONS, + RETRYABLE_REASONS, + EventDisposition, + IngestionReasonCode, + is_permanent, + is_retryable, +) +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import ( + BatchReceipt, + EventAck, + IngestionContext, + IngestionIssue, + IngestionStore, + ResearchEventRecord, + TelemetryBatchAckV1, + TelemetryBatchRequestV1, +) +from .service import CapabilityVerifier, compute_event_digest, ingest_batch +from .store import FakeIngestionStore, SqlAlchemyIngestionStore, row_to_record +from .validation import validate_batch_event + +__all__ = [ + "PERMANENT_REASONS", + "RETRYABLE_REASONS", + "BatchReceipt", + "CapabilityVerifier", + "EventAck", + "EventDisposition", + "FakeIngestionStore", + "IngestionContext", + "IngestionIssue", + "IngestionReasonCode", + "IngestionStore", + "IntegrityConflictError", + "ResearchEventRecord", + "ReceiptConflictError", + "SqlAlchemyIngestionStore", + "StoreUnavailable", + "TelemetryBatchAckV1", + "TelemetryBatchRequestV1", + "compute_event_digest", + "ingest_batch", + "is_permanent", + "is_retryable", + "row_to_record", + "validate_batch_event", +] diff --git a/src/research/telemetry/ingestion/enums.py b/src/research/telemetry/ingestion/enums.py new file mode 100644 index 0000000..11c658b --- /dev/null +++ b/src/research/telemetry/ingestion/enums.py @@ -0,0 +1,101 @@ +"""Closed vocabularies for idempotent telemetry ingestion (Issue 09). + +A disposition is terminal for one event: ``REJECTED`` reasons are permanent +(the event must not be retried) while ``RETRYABLE`` reasons are transient. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "EventDisposition", + "IngestionReasonCode", + "PERMANENT_REASONS", + "RETRYABLE_REASONS", + "is_permanent", + "is_retryable", +] + + +class EventDisposition(str, Enum): + """Terminal per-event acknowledgement disposition.""" + + ACCEPTED = "ACCEPTED" + DUPLICATE = "DUPLICATE" + REJECTED = "REJECTED" + RETRYABLE = "RETRYABLE" + + +class IngestionReasonCode(str, Enum): + """Stable machine-readable reason for an event disposition.""" + + OK = "OK" + + # Permanent (the event must not be retried). + INVALID_SCHEMA = "INVALID_SCHEMA" + UNKNOWN_SCHEMA_VERSION = "UNKNOWN_SCHEMA_VERSION" + MISSING_PROVENANCE = "MISSING_PROVENANCE" + MISSING_EVENT_ID = "MISSING_EVENT_ID" + CONTEXT_MISMATCH = "CONTEXT_MISMATCH" + SESSION_OUT_OF_SCOPE = "SESSION_OUT_OF_SCOPE" + SESSION_TERMINAL = "SESSION_TERMINAL" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + REVOKED = "REVOKED" + INTEGRITY_CONFLICT = "INTEGRITY_CONFLICT" + SENSITIVE_PAYLOAD = "SENSITIVE_PAYLOAD" + PRIVACY_BLOCKED = "PRIVACY_BLOCKED" + EVENT_SEQUENCE_GAP = "EVENT_SEQUENCE_GAP" + RECEIPT_CONFLICT = "RECEIPT_CONFLICT" + DUPLICATE_BATCH = "DUPLICATE_BATCH" + BATCH_TOO_LARGE = "BATCH_TOO_LARGE" + CONTINUITY_REQUIRED = "CONTINUITY_REQUIRED" + + # Transient (safe to retry after re-bootstrap or backoff). + STORE_UNAVAILABLE = "STORE_UNAVAILABLE" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + # Operator kill switch: new batches are not stored while engaged. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + + +# Reason codes that permanently reject an event. +PERMANENT_REASONS = frozenset( + { + IngestionReasonCode.INVALID_SCHEMA, + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + IngestionReasonCode.MISSING_PROVENANCE, + IngestionReasonCode.MISSING_EVENT_ID, + IngestionReasonCode.CONTEXT_MISMATCH, + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + IngestionReasonCode.SESSION_TERMINAL, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + IngestionReasonCode.REVOKED, + IngestionReasonCode.INTEGRITY_CONFLICT, + IngestionReasonCode.SENSITIVE_PAYLOAD, + IngestionReasonCode.PRIVACY_BLOCKED, + IngestionReasonCode.EVENT_SEQUENCE_GAP, + IngestionReasonCode.RECEIPT_CONFLICT, + IngestionReasonCode.DUPLICATE_BATCH, + IngestionReasonCode.BATCH_TOO_LARGE, + IngestionReasonCode.CONTINUITY_REQUIRED, + } +) + +# Reason codes that permit a later retry. +RETRYABLE_REASONS = frozenset( + { + IngestionReasonCode.STORE_UNAVAILABLE, + IngestionReasonCode.CAPABILITY_INVALID, + IngestionReasonCode.KILL_SWITCH_ENGAGED, + } +) + + +def is_permanent(reason: IngestionReasonCode) -> bool: + """Whether ``reason`` permanently rejects an event.""" + return reason in PERMANENT_REASONS + + +def is_retryable(reason: IngestionReasonCode) -> bool: + """Whether ``reason`` permits a later retry.""" + return reason in RETRYABLE_REASONS diff --git a/src/research/telemetry/ingestion/errors.py b/src/research/telemetry/ingestion/errors.py new file mode 100644 index 0000000..15afe8a --- /dev/null +++ b/src/research/telemetry/ingestion/errors.py @@ -0,0 +1,48 @@ +"""Typed errors for ingestion store failures.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "IntegrityConflictError", + "ReceiptConflictError", + "StoreUnavailable", +] + + +class StoreUnavailable(RuntimeError): + """Raised when the event store is transiently unavailable. + + The caller must return a retryable acknowledgement, never a false accept. + """ + + +class ReceiptConflictError(RuntimeError): + """Raised when a concurrent commit already stored a receipt for the batch. + + The batch id is unique, so exactly one commit wins. The loser must discard + its own staged writes and return the winner's immutable receipt. + """ + + def __init__(self, batch_id: "UUID") -> None: + super().__init__(f"batch {batch_id} already has a stored receipt") + self.batch_id = batch_id + + +class IntegrityConflictError(RuntimeError): + """Raised when an event id already exists with a different digest.""" + + def __init__( + self, event_id: UUID, stored_digest: str, presented_digest: str + ) -> None: + super().__init__( + f"integrity conflict for event {event_id}: stored digest differs from " + "the presented digest" + ) + self.event_id = event_id + self.stored_digest = stored_digest + self.presented_digest = presented_digest diff --git a/src/research/telemetry/ingestion/models.py b/src/research/telemetry/ingestion/models.py new file mode 100644 index 0000000..2c5558b --- /dev/null +++ b/src/research/telemetry/ingestion/models.py @@ -0,0 +1,216 @@ +"""Pydantic v2 contracts for idempotent telemetry ingestion (Issue 09). + +``TelemetryBatchRequestV1`` carries already-privacy-filtered canonical events +plus a scoped session capability. ``TelemetryBatchAckV1`` groups event ids by +disposition; an event id never appears in two groups, and a missing +acknowledgement is never treated as accepted. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional, Protocol, Sequence +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from research.runtime.bootstrap.models import ( + SessionCapability, # noqa: TC001 - pydantic resolves model annotations at runtime +) +from research.telemetry.models import ( + CanonicalEventV1, # noqa: TC001 - pydantic resolves model annotations at runtime +) + +from .enums import ( + EventDisposition, # noqa: TC001 - pydantic resolves model field annotations at runtime + IngestionReasonCode, # noqa: TC001 - pydantic resolves model field annotations at runtime +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class TelemetryBatchRequestV1(BaseModel): + """One upload request from the durable local spool.""" + + model_config = _BASE + + batch_id: UUID + protocol_version: str = "1" + telemetry_schema_version: str = "1" + session_capability: SessionCapability + events: list[CanonicalEventV1] = Field(default_factory=list) + client_instance_id: str + previous_ack_cursor: Optional[str] = None + + @property + def size(self) -> int: + """Number of events in the batch.""" + return len(self.events) + + +class EventAck(BaseModel): + """Per-event acknowledgement.""" + + model_config = _BASE + + event_id: Optional[UUID] = None + disposition: EventDisposition + reason: Optional[IngestionReasonCode] = None + retry_hint: Optional[int] = None + stored_digest: Optional[str] = None + + +class TelemetryBatchAckV1(BaseModel): + """Durable batch acknowledgement, safe to retry and deterministic per event.""" + + model_config = _BASE + + receipt_id: UUID + batch_id: UUID + server_time: datetime + accepted: list[EventAck] = Field(default_factory=list) + duplicate: list[EventAck] = Field(default_factory=list) + rejected: list[EventAck] = Field(default_factory=list) + retryable: list[EventAck] = Field(default_factory=list) + retry_hint: Optional[int] = None + # Coverage diagnostics (e.g. per-emitter sequence gaps) keyed by event id. + coverage_diagnostics: dict[str, str] = Field(default_factory=dict) + + @model_validator(mode="after") + def _unique_event_ids(self) -> TelemetryBatchAckV1: + seen: set[UUID] = set() + for group in (self.accepted, self.duplicate, self.rejected, self.retryable): + for ack in group: + if ack.event_id is None: + continue + if ack.event_id in seen: + raise ValueError( + f"event {ack.event_id} appears in more than one ack group" + ) + seen.add(ack.event_id) + return self + + def acknowledged_ids(self) -> list[UUID]: + """Ids the spool may delete: accepted + duplicate.""" + return [ack.event_id for ack in self.accepted + self.duplicate if ack.event_id] + + @property + def is_empty(self) -> bool: + """Whether the acknowledgement carries no per-event dispositions.""" + return not (self.accepted or self.duplicate or self.rejected or self.retryable) + + +class IngestionContext(BaseModel): + """Authorized, resolved context every accepted event is bound to. + + Every identity field is derived from the authorized enrollment/session, never + from the client event payload. + """ + + model_config = _FROZEN + + study_id: UUID + enrollment_id: UUID + study_revision_id: UUID + research_session_id: UUID + revocation_epoch: int = 0 + + +class IngestionIssue(BaseModel): + """One typed validation finding for a batch event.""" + + model_config = _BASE + + code: IngestionReasonCode + field: str = "" + message: str = "" + permanent: bool = True + + +class ResearchEventRecord(BaseModel): + """One immutable accepted canonical event fact. + + ``envelope`` is the complete validated ``CanonicalEventV1`` document and is + the event authority: it preserves correlations, monotonic time, lifecycle, + unknown-value fields, provenance, coverage and the privacy summary. The + remaining fields are the searchable scalar columns derived from that same + envelope in the same transaction (identity/ordering for indexes and joins); + they are never a second, independently writable copy of the content. + Ingestion metadata (``digest``, ``accepted_at``, ``retention_state``) lives + outside the envelope. + """ + + model_config = _FROZEN + + event_id: UUID + schema_version: str + event_type: str + source: str + study_id: Optional[UUID] = None + revision_id: Optional[UUID] = None + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + agent_run_id: Optional[str] = None + emitter_id: str + emitter_sequence: int + occurred_at: datetime + # The complete validated canonical envelope (the authority). + envelope: dict[str, Any] = Field(default_factory=dict) + digest: str + accepted_at: datetime + retention_state: str = "RETAINED" + + +class BatchReceipt(BaseModel): + """An immutable, retry-safe batch acknowledgement record.""" + + model_config = _FROZEN + + receipt_id: UUID + batch_id: UUID + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + accepted_at: datetime + ack: TelemetryBatchAckV1 + + +class IngestionStore(Protocol): + """The persistence surface the ingestion service depends on. + + The service owns the transaction boundary: ``insert_events`` and + ``record_batch_receipt`` stage their writes, and ``commit``/``rollback`` + decide whether the whole batch (accepted events + immutable receipt) becomes + durable. A batch therefore commits in full or not at all. + """ + + def get_stored_digests( + self, event_ids: Sequence[UUID] + ) -> dict[UUID, str]: # pragma: no cover - structural protocol + ... + + def insert_events( + self, records: Sequence[ResearchEventRecord] + ) -> list[UUID]: # pragma: no cover - structural protocol + ... + + def record_batch_receipt( + self, receipt: BatchReceipt + ) -> None: # pragma: no cover - structural protocol + ... + + def get_receipt( + self, batch_id: UUID + ) -> Optional[BatchReceipt]: # pragma: no cover - structural protocol + ... + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[UUID] + ) -> Optional[int]: # pragma: no cover - structural protocol + ... + + def commit(self) -> None: # pragma: no cover - structural protocol + ... + + def rollback(self) -> None: # pragma: no cover - structural protocol + ... diff --git a/src/research/telemetry/ingestion/service.py b/src/research/telemetry/ingestion/service.py new file mode 100644 index 0000000..dc1f3c8 --- /dev/null +++ b/src/research/telemetry/ingestion/service.py @@ -0,0 +1,738 @@ +"""Idempotent batch ingestion service (Issue 09). + +Authorization happens inside the same transaction that persists the batch: the +scoped session capability is verified (signature, audience, scope, expiry, +revocation epoch), the enrollment and research session rows are locked +(``SELECT ... FOR UPDATE``) and validated, the events are classified, and the +accepted events plus the immutable batch receipt are committed together. The +ACK is returned only after commit, so a mid-batch database error rolls the whole +batch back and produces a retryable result — never a partial accept. + +Per event, classification is deterministic given store state: + +* same ``event_id`` + same digest already stored -> ``DUPLICATE`` (acknowledged, + never a second fact); +* same ``event_id`` + different digest -> ``REJECTED`` integrity conflict, never + overwritten; +* invalid/unsafe/out-of-scope/blocked -> ``REJECTED`` with a permanent typed + reason; +* transient store failure -> ``RETRYABLE`` with a hint and no false ACK; +* otherwise -> ``ACCEPTED``. + +The returned acknowledgement is immutable and retry-safe: re-POSTing the same +``batch_id`` returns the stored receipt unchanged, and a concurrent duplicate +loses the receipt race but returns the winner's receipt (never a second one). +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Callable, Optional, Protocol, Sequence + +from research.canonical import canonical_hash +from research.participants.enums import EnrollmentStatus +from research.runtime.bootstrap.models import CapabilityReasonCode + +from .enums import EventDisposition, IngestionReasonCode +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import ( + BatchReceipt, + EventAck, + IngestionContext, + IngestionStore, + ResearchEventRecord, + TelemetryBatchAckV1, + TelemetryBatchRequestV1, +) +from .validation import validate_batch_event + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from research.runtime.bootstrap.models import CapabilityVerification, SessionCapability + from research.runtime.sessions.models import ResearchSessionV1 + from research.telemetry.models import CanonicalEventV1 + +__all__ = ["CapabilityVerifier", "compute_event_digest", "ingest_batch"] + +DEFAULT_RETRY_HINT_SECONDS = 30 +DEFAULT_MAX_EVENTS = 500 + + +class CapabilityVerifier(Protocol): + """Verifies a session capability against its resolved subject. + + The resolved ``(enrollment, session, revision)`` are passed so the verifier + can reject a cryptographically valid capability that was issued for a + different subject. Verifiers may accept them as optional keyword arguments + (the default ``None`` skips the subject comparison). + """ + + def __call__( + self, + capability: SessionCapability, + *, + now: datetime, + current_revocation_epoch: int, + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_revision_id: Optional[uuid.UUID] = None, + ) -> CapabilityVerification: # pragma: no cover - structural protocol + ... + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def compute_event_digest(event: CanonicalEventV1) -> str: + """Content digest of a canonical event (stable, order-independent).""" + return canonical_hash(event.model_dump(mode="json")) + + +def _record_from_event( + event: CanonicalEventV1, + context: IngestionContext, + digest: str, + accepted_at: datetime, +) -> ResearchEventRecord: + """Build the stored record from the *authorized* context. + + ``study_id``/``revision_id``/``enrollment_id``/``research_session_id`` are + always taken from the authorized enrollment/session, never from the client + event payload. A payload field that disagrees with the context is rejected + by :func:`validate_batch_event` before this runs. + """ + return ResearchEventRecord( + event_id=event.event_id, + schema_version=event.schema_version, + event_type=event.event_type, + source=event.source, + study_id=context.study_id, + revision_id=context.study_revision_id, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + agent_run_id=event.agent_run_id, + emitter_id=event.emitter_id, + emitter_sequence=event.emitter_sequence, + occurred_at=event.occurred_at, + # The complete validated envelope is the stored authority; the scalar + # fields above are searchable projections derived from it. + envelope=event.model_dump(mode="json"), + digest=digest, + accepted_at=accepted_at, + retention_state="RETAINED", + ) + + +def _ack( + batch_id: uuid.UUID, + server_time: datetime, + *, + accepted: Sequence[EventAck] = (), + duplicate: Sequence[EventAck] = (), + rejected: Sequence[EventAck] = (), + retryable: Sequence[EventAck] = (), + retry_hint: Optional[int] = None, + diagnostics: Optional[dict[str, str]] = None, +) -> TelemetryBatchAckV1: + return TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=batch_id, + server_time=server_time, + accepted=list(accepted), + duplicate=list(duplicate), + rejected=list(rejected), + retryable=list(retryable), + retry_hint=retry_hint, + coverage_diagnostics=dict(diagnostics or {}), + ) + + +def _retryable_ack(ack: TelemetryBatchAckV1) -> TelemetryBatchAckV1: + """Re-target every accepted event as retryable (no false ACK).""" + retryable = [ + EventAck( + event_id=event_ack.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.STORE_UNAVAILABLE, + ) + for event_ack in ack.accepted + ] + return TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=ack.batch_id, + server_time=ack.server_time, + duplicate=list(ack.duplicate), + rejected=list(ack.rejected), + retryable=list(ack.retryable) + retryable, + retry_hint=ack.retry_hint, + coverage_diagnostics=dict(ack.coverage_diagnostics), + ) + + +def _finalize( + store: IngestionStore, + batch_id: uuid.UUID, + ack: TelemetryBatchAckV1, + *, + records: Sequence[ResearchEventRecord] = (), + enrollment_id: Optional[uuid.UUID], + research_session_id: Optional[uuid.UUID], + retry_hint: Optional[int] = None, +) -> TelemetryBatchAckV1: + """Persist the accepted events and the immutable receipt in one transaction. + + A retryable batch is not cached (a retry must be retryable): any row locks + acquired for the batch are released with a rollback. A terminal decision + stages the events and receipt, commits once, and only then returns the ACK. + + An :class:`IntegrityConflictError` identifies a single poison event; it is + terminally rejected and the remaining records are persisted in a fresh + transaction, so one bad event cannot make its siblings retryable forever. + """ + if ack.retryable: + store.rollback() + return ack + + receipt = BatchReceipt( + receipt_id=ack.receipt_id, + batch_id=batch_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + accepted_at=ack.server_time, + ack=ack, + ) + try: + if records: + store.insert_events(records) + store.record_batch_receipt(receipt) + store.commit() + except ReceiptConflictError: + # A concurrent submission already committed a receipt for this batch: + # never a second one. Return the winner's immutable receipt. + store.rollback() + existing = store.get_receipt(batch_id) + return existing.ack if existing is not None else _retryable_ack(ack) + except IntegrityConflictError as conflict: + store.rollback() + # A single poison event (its ``(session, emitter, sequence)`` key or its + # event-id/digest identity is already taken) must not wedge the whole + # batch: terminally reject only that event and persist the survivors. + # Each recursion removes exactly one record, so this is bounded by the + # batch size and always terminates. + survivors = [ + record for record in records if record.event_id != conflict.event_id + ] + if len(survivors) == len(records): + # The conflict is not one of the staged records (for example a + # concurrent commit we cannot attribute): never loop, and never + # falsely acknowledge; fall back to a retryable batch. + return _retryable_ack(ack) + rejected = list(ack.rejected) + [ + EventAck( + event_id=conflict.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.INTEGRITY_CONFLICT, + stored_digest=conflict.stored_digest, + ) + ] + accepted_survivors = [ + event_ack + for event_ack in ack.accepted + if event_ack.event_id != conflict.event_id + ] + conflict_ack = TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=ack.batch_id, + server_time=ack.server_time, + accepted=accepted_survivors, + duplicate=list(ack.duplicate), + rejected=rejected, + retryable=list(ack.retryable), + retry_hint=retry_hint, + coverage_diagnostics=dict(ack.coverage_diagnostics), + ) + return _finalize( + store, + batch_id, + conflict_ack, + records=survivors, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + retry_hint=retry_hint, + ) + except StoreUnavailable: + store.rollback() + return _retryable_ack(ack) + + # The unique ``batch_id`` guarantees one owner. If a concurrent submission + # won, return its stored receipt rather than this (unstored) ACK. + stored = store.get_receipt(batch_id) + if stored is not None and stored.receipt_id != ack.receipt_id: + return stored.ack + return ack + + +def _reject_all( + store: IngestionStore, + batch_id: uuid.UUID, + events: Sequence[CanonicalEventV1], + server_time: datetime, + reason: IngestionReasonCode, + *, + disposition: EventDisposition, + retry_hint: Optional[int] = None, + enrollment_id: Optional[uuid.UUID] = None, + research_session_id: Optional[uuid.UUID] = None, +) -> TelemetryBatchAckV1: + group = [ + EventAck(event_id=event.event_id, disposition=disposition, reason=reason) + for event in events + ] + if disposition == EventDisposition.RETRYABLE: + ack = _ack(batch_id, server_time, retryable=group, retry_hint=retry_hint) + else: + ack = _ack(batch_id, server_time, rejected=group) + return _finalize( + store, + batch_id, + ack, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + retry_hint=retry_hint, + ) + + +def _anchor_context( + request: TelemetryBatchRequestV1, + enrollment_resolver: Callable[[uuid.UUID], Optional[Enrollment]], + session_resolver: Callable[[uuid.UUID], Optional[ResearchSessionV1]], +) -> Optional[tuple[IngestionContext, Enrollment, ResearchSessionV1]]: + """Resolve (and lock) the authorized subject of the batch. + + The resolvers are expected to lock the enrollment and session rows + (``SELECT ... FOR UPDATE``); this serializes concurrent batches for the same + subject, which is what makes batch idempotency concurrency-safe. + """ + for event in request.events: + if event.research_session_id is None or event.enrollment_id is None: + continue + session = session_resolver(event.research_session_id) + if session is None: + continue + enrollment = enrollment_resolver(event.enrollment_id) + if enrollment is None: + continue + if session.enrollment_id != enrollment.enrollment_id: + continue + if enrollment.study_revision_id != session.study_revision_id: + continue + context = IngestionContext( + study_id=enrollment.study_id, + enrollment_id=enrollment.enrollment_id, + study_revision_id=session.study_revision_id, + research_session_id=session.research_session_id, + revocation_epoch=enrollment.revocation_epoch, + ) + return context, enrollment, session + return None + + +def _is_terminal(session: ResearchSessionV1) -> bool: + state = session.state + return state.is_terminal if hasattr(state, "is_terminal") else False + + +def ingest_batch( + request: TelemetryBatchRequestV1, + *, + capability_verifier: CapabilityVerifier, + enrollment_resolver: Callable[[uuid.UUID], Optional[Enrollment]], + session_resolver: Callable[[uuid.UUID], Optional[ResearchSessionV1]], + store: IngestionStore, + now: Optional[datetime] = None, + max_events: int = DEFAULT_MAX_EVENTS, + continuity_required: bool = False, + retry_hint: int = DEFAULT_RETRY_HINT_SECONDS, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> TelemetryBatchAckV1: + """Ingest one batch and return a durable, deterministic acknowledgement.""" + server_time = _now(now) + + # Retry of an already-received batch returns the immutable receipt. The + # authoritative re-check happens after the subject rows are locked. + existing_receipt = store.get_receipt(request.batch_id) + if existing_receipt is not None: + return existing_receipt.ack + + if len(request.events) > max_events: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.BATCH_TOO_LARGE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + ) + + # Operator kill switch (Issue 13): accepted batches are not stored while + # engaged. The check is injected so ingestion keeps no operations dependency. + if kill_switch_check is not None and kill_switch_check(): + kill_switch_group = [ + EventAck( + event_id=event.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.KILL_SWITCH_ENGAGED, + ) + for event in request.events + ] + return _ack( + request.batch_id, server_time, retryable=kill_switch_group, retry_hint=retry_hint + ) + + if not request.events: + ack = _ack(request.batch_id, server_time) + return _finalize( + store, request.batch_id, ack, enrollment_id=None, research_session_id=None + ) + + anchored = _anchor_context(request, enrollment_resolver, session_resolver) + if anchored is None: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + ) + context, enrollment, session = anchored + + # Re-check after acquiring the subject locks: a concurrent submission of the + # same batch_id has now either committed (return its receipt) or is blocked + # behind the same lock. + locked_receipt = store.get_receipt(request.batch_id) + if locked_receipt is not None: + store.rollback() + return locked_receipt.ack + + verification = capability_verifier( + request.session_capability, + now=server_time, + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=context.enrollment_id, + expected_research_session_id=context.research_session_id, + expected_revision_id=context.study_revision_id, + ) + if not verification.ok: + if verification.reason == CapabilityReasonCode.REVOKED: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.REVOKED, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.CAPABILITY_INVALID, + disposition=EventDisposition.RETRYABLE, + retry_hint=retry_hint, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + if _is_terminal(session): + # ENDED/REVOKED sessions are terminal: no later event is ever stored. + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.SESSION_TERMINAL, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + return _ingest_authorized_events( + store, + batch_id=request.batch_id, + events=request.events, + context=context, + telemetry_schema_version=request.telemetry_schema_version, + server_time=server_time, + continuity_required=continuity_required, + retry_hint=retry_hint, + ) + + +def _ingest_authorized_events( + store: IngestionStore, + *, + batch_id: uuid.UUID, + events: Sequence[CanonicalEventV1], + context: IngestionContext, + telemetry_schema_version: str, + server_time: datetime, + continuity_required: bool, + retry_hint: int, +) -> TelemetryBatchAckV1: + """Persist already-authorized events with the HTTP route's exact semantics. + + Shared by the participant upload route and the server-authorized internal + entry point: identical per-event validation, digest/uniqueness handling, + sequence continuity, receipt and commit-before-ACK. The caller has already + verified the capability (route) or derived the context server-side + (internal adapter) and applied the terminal/enrollment rules. + """ + accepted: list[EventAck] = [] + duplicate: list[EventAck] = [] + rejected: list[EventAck] = [] + retryable: list[EventAck] = [] + diagnostics: dict[str, str] = {} + candidates: list[tuple[CanonicalEventV1, str]] = [] + + try: + stored = store.get_stored_digests([event.event_id for event in events]) + except StoreUnavailable: + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.STORE_UNAVAILABLE, + disposition=EventDisposition.RETRYABLE, + retry_hint=retry_hint, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + ) + + for event in events: + issues = validate_batch_event( + event, + context, + telemetry_schema_version=telemetry_schema_version, + ) + if issues: + code = issues[0].code + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=code, + ) + ) + continue + + digest = compute_event_digest(event) + existing_digest = stored.get(event.event_id) + if existing_digest is not None: + if existing_digest == digest: + duplicate.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.DUPLICATE, + stored_digest=existing_digest, + ) + ) + else: + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.INTEGRITY_CONFLICT, + stored_digest=existing_digest, + ) + ) + continue + + candidates.append((event, digest)) + + # Sequence continuity: a gap is accepted with a coverage diagnostic unless + # the protocol declares continuity required. + sequenced: list[tuple[CanonicalEventV1, str]] = [] + in_batch_last: dict[tuple[str, Optional[uuid.UUID]], int] = {} + for event, digest in candidates: + key = (event.emitter_id, event.research_session_id or context.research_session_id) + cursor = store.get_emitter_cursor(key[0], key[1]) + last = max(cursor or 0, in_batch_last.get(key, 0)) + expected = last + 1 + if event.emitter_sequence != expected: + if continuity_required: + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.CONTINUITY_REQUIRED, + ) + ) + in_batch_last[key] = max(last, event.emitter_sequence) + continue + diagnostics[str(event.event_id)] = ( + f"expected emitter sequence {expected}, received " + f"{event.emitter_sequence}" + ) + accepted.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.ACCEPTED, + reason=IngestionReasonCode.EVENT_SEQUENCE_GAP, + stored_digest=digest, + ) + ) + else: + accepted.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.ACCEPTED, + stored_digest=digest, + ) + ) + in_batch_last[key] = max(last, event.emitter_sequence) + sequenced.append((event, digest)) + + records = [ + _record_from_event(event, context, digest, server_time) + for event, digest in sequenced + ] + ack = _ack( + batch_id, + server_time, + accepted=accepted, + duplicate=duplicate, + rejected=rejected, + retryable=retryable, + retry_hint=retry_hint, + diagnostics=diagnostics, + ) + return _finalize( + store, + batch_id, + ack, + records=records, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + +def ingest_events_for_context( + *, + context: IngestionContext, + enrollment: Enrollment, + session: ResearchSessionV1, + events: Sequence[CanonicalEventV1], + store: IngestionStore, + now: Optional[datetime] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, + continuity_required: bool = False, + telemetry_schema_version: str = "1", + max_events: int = DEFAULT_MAX_EVENTS, + retry_hint: int = DEFAULT_RETRY_HINT_SECONDS, +) -> TelemetryBatchAckV1: + """Server-authorized internal ingestion entry point (no HTTP hop). + + A narrow adapter calls this directly with a context it has already derived + from the authorized account + frozen assignment (an explicit research + binding), so legacy producers stop writing a parallel store. It applies the + same kill-switch, enrollment-ACTIVE and terminal-session rules as the + participant upload route, then shares the route's validation, uniqueness, + receipt and commit-before-ACK semantics. The context/assignment is trusted + server-side state, never a caller-supplied body. + """ + server_time = _now(now) + batch_id = uuid.uuid4() + + if len(events) > max_events: + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.BATCH_TOO_LARGE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + ) + + if kill_switch_check is not None and kill_switch_check(): + group = [ + EventAck( + event_id=event.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.KILL_SWITCH_ENGAGED, + ) + for event in events + ] + return _ack(batch_id, server_time, retryable=group, retry_hint=retry_hint) + + if not events: + ack = _ack(batch_id, server_time) + return _finalize( + store, + batch_id, + ack, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + if _is_terminal(session): + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.SESSION_TERMINAL, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + return _ingest_authorized_events( + store, + batch_id=batch_id, + events=events, + context=context, + telemetry_schema_version=telemetry_schema_version, + server_time=server_time, + continuity_required=continuity_required, + retry_hint=retry_hint, + ) diff --git a/src/research/telemetry/ingestion/store.py b/src/research/telemetry/ingestion/store.py new file mode 100644 index 0000000..48c1c5f --- /dev/null +++ b/src/research/telemetry/ingestion/store.py @@ -0,0 +1,373 @@ +"""Persistence helpers and an in-memory test double for ingestion. + +The SQLAlchemy store takes a caller-supplied ``Session`` and stages its writes: +``insert_events`` and ``record_batch_receipt`` never commit, so the ingestion +service can persist the accepted events and the immutable batch receipt in one +transaction and return the ACK only after ``commit``. ``FakeIngestionStore`` is +a pure in-memory double that mirrors the same commit/rollback semantics (and +enforces unique event ids/digest conflicts), so atomicity is machine-testable +without PostgreSQL. +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Optional, Sequence + +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError, SQLAlchemyError + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +from database.research_schemas import ( + ResearchEvent, + TelemetryBatchReceipt, +) + +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import BatchReceipt, IngestionStore, ResearchEventRecord + +__all__ = [ + "FakeIngestionStore", + "IngestionStore", + "SqlAlchemyIngestionStore", + "get_receipt_by_id", + "row_to_record", +] + + +class FakeIngestionStore: + """In-memory ingestion store for deterministic tests. + + Writes are staged until :meth:`commit`, so a failed commit (or a mid-batch + error) leaves the committed state untouched. ``fail_inserts`` simulates a + transient failure while staging events, ``fail_receipts`` while staging the + receipt and ``fail_commit`` at commit time. + """ + + def __init__(self) -> None: + self._events: dict[uuid.UUID, ResearchEventRecord] = {} + self._receipts: dict[uuid.UUID, BatchReceipt] = {} + self._staged_events: dict[uuid.UUID, ResearchEventRecord] = {} + self._staged_receipts: dict[uuid.UUID, BatchReceipt] = {} + self.fail_inserts = False + self.fail_receipts = False + self.fail_commit = False + + # -- events ----------------------------------------------------------- + + def get_stored_digests( + self, event_ids: Sequence[uuid.UUID] + ) -> dict[uuid.UUID, str]: + """Return ``{event_id: digest}`` for event ids already committed.""" + return { + event_id: self._events[event_id].digest + for event_id in event_ids + if event_id in self._events + } + + def _emitter_key_conflict( + self, record: ResearchEventRecord + ) -> Optional[ResearchEventRecord]: + """Return a committed/staged record colliding on the emitter key. + + Mirrors the database's ``uq_research_event_session_emitter_sequence`` + unique constraint: one ``(research_session_id, emitter_id, + emitter_sequence)`` may identify at most one event id. + """ + for existing in (*self._events.values(), *self._staged_events.values()): + if ( + existing.event_id != record.event_id + and existing.emitter_id == record.emitter_id + and existing.research_session_id == record.research_session_id + and existing.emitter_sequence == record.emitter_sequence + ): + return existing + return None + + def insert_events(self, records: Sequence[ResearchEventRecord]) -> list[uuid.UUID]: + """Stage records, enforcing unique ids, digests, and emitter keys. + + A same-id/same-digest insert is a no-op (idempotent); a same-id/different + digest, or a different event id reusing the same + ``(session, emitter, sequence)`` key, raises + :class:`IntegrityConflictError`. ``fail_inserts`` simulates a transiently + unavailable store. + """ + if self.fail_inserts: + raise StoreUnavailable("in-memory store is unavailable") + + inserted: list[uuid.UUID] = [] + for record in records: + existing = self._events.get(record.event_id) + if existing is not None: + if existing.digest != record.digest: + raise IntegrityConflictError( + record.event_id, existing.digest, record.digest + ) + continue + emitter_conflict = self._emitter_key_conflict(record) + if emitter_conflict is not None: + # The stored digest is the one that already owns this emitter + # key; the presented record is the poison event. + raise IntegrityConflictError( + record.event_id, emitter_conflict.digest, record.digest + ) + self._staged_events[record.event_id] = record + inserted.append(record.event_id) + return inserted + + def stored_events(self) -> list[ResearchEventRecord]: + """Return the committed records (test introspection).""" + return list(self._events.values()) + + def event_count(self) -> int: + """Number of committed canonical facts.""" + return len(self._events) + + # -- receipts --------------------------------------------------------- + + def record_batch_receipt(self, receipt: BatchReceipt) -> None: + """Stage a receipt; the committed receipt for a batch is immutable.""" + if self.fail_receipts: + raise StoreUnavailable("in-memory receipt store is unavailable") + self._staged_receipts[receipt.batch_id] = receipt + + def get_receipt(self, batch_id: uuid.UUID) -> Optional[BatchReceipt]: + """Fetch the committed immutable receipt for a batch, or ``None``.""" + return self._receipts.get(batch_id) + + # -- transaction boundary -------------------------------------------- + + def commit(self) -> None: + """Make every staged write durable, atomically.""" + if self.fail_commit: + raise StoreUnavailable("in-memory commit failed") + # The batch id is unique: a committed receipt for the same batch wins + # and this commit must not replace it. + for batch_id, receipt in self._staged_receipts.items(): + existing = self._receipts.get(batch_id) + if existing is not None and existing.receipt_id != receipt.receipt_id: + raise ReceiptConflictError(batch_id) + for key, record in self._staged_events.items(): + self._events.setdefault(key, record) + for batch_id, receipt in self._staged_receipts.items(): + self._receipts.setdefault(batch_id, receipt) + self._staged_events.clear() + self._staged_receipts.clear() + + def rollback(self) -> None: + """Discard every staged write, leaving committed state unchanged.""" + self._staged_events.clear() + self._staged_receipts.clear() + + # -- emitter cursors -------------------------------------------------- + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[uuid.UUID] + ) -> Optional[int]: + """Return the max committed sequence for an emitter, or ``None``.""" + sequences = [ + record.emitter_sequence + for record in self._events.values() + if record.emitter_id == emitter_id + and record.research_session_id == research_session_id + ] + return max(sequences) if sequences else None + + +def row_to_record(row: ResearchEvent) -> ResearchEventRecord: + """Rehydrate a stored event row into the record model. + + The complete ``envelope_json`` document is the authority; the searchable + scalar columns are read alongside it (they are derived from the same envelope + at insert time). + """ + data = { + name: getattr(row, name) + for name in ResearchEventRecord.model_fields + if name != "envelope" + } + data["envelope"] = row.envelope_json or {} + return ResearchEventRecord.model_validate(data) + + +class SqlAlchemyIngestionStore: + """SQLAlchemy-backed ingestion store with an explicit transaction boundary.""" + + def __init__(self, session: Session) -> None: + self.session = session + + def get_stored_digests( + self, event_ids: Sequence[uuid.UUID] + ) -> dict[uuid.UUID, str]: + """Return ``{event_id: digest}`` for already-stored event ids.""" + if not event_ids: + return {} + statement = select(ResearchEvent.event_id, ResearchEvent.digest).where( + ResearchEvent.event_id.in_(list(event_ids)) + ) + return {row[0]: row[1] for row in self.session.execute(statement).all()} + + def _emitter_key_conflict( + self, records: Sequence[ResearchEventRecord] + ) -> Optional[IntegrityConflictError]: + """Attribute a unique-key violation to the exact poison record. + + Called only after a failed insert has been rolled back, so the probe runs + in a fresh transaction. Returns ``None`` when no record collides on the + ``(research_session_id, emitter_id, emitter_sequence)`` unique key. + """ + for record in records: + statement = select(ResearchEvent.event_id, ResearchEvent.digest).where( + ResearchEvent.research_session_id == record.research_session_id, + ResearchEvent.emitter_id == record.emitter_id, + ResearchEvent.emitter_sequence == record.emitter_sequence, + ResearchEvent.event_id != record.event_id, + ) + row = self.session.execute(statement).first() + if row is not None: + return IntegrityConflictError(record.event_id, row[1], record.digest) + return None + + def insert_events(self, records: Sequence[ResearchEventRecord]) -> list[uuid.UUID]: + """Stage insert-only event persistence (unique ``event_id``). + + ``ON CONFLICT DO NOTHING`` makes a concurrent duplicate a no-op rather + than aborting the transaction, so the digest-identity check (performed + inside the same transaction) remains authoritative and an existing fact + is never overwritten. + + A violation of the ``(research_session_id, emitter_id, + emitter_sequence)`` unique key is not covered by the ``event_id`` + conflict target; it aborts the statement. Rather than reporting the whole + batch retryable (which would wedge every later retry), the conflict is + attributed to the exact event so the service can terminally reject only + that poison event and persist its siblings. + """ + if not records: + return [] + try: + values = [ + { + **record.model_dump(exclude={"envelope"}), + "envelope_json": record.envelope, + } + for record in records + ] + statement = pg_insert(ResearchEvent).values(values) + statement = statement.on_conflict_do_nothing( + index_elements=["event_id"] + ) + self.session.execute(statement) + except IntegrityError as error: + self.session.rollback() + conflict = self._emitter_key_conflict(records) + if conflict is not None: + raise conflict from error + raise StoreUnavailable("event store is unavailable") from error + except SQLAlchemyError as error: + raise StoreUnavailable("event store is unavailable") from error + return [record.event_id for record in records] + + def record_batch_receipt(self, receipt: BatchReceipt) -> None: + """Stage a receipt if the batch has none (immutable, retry-safe). + + ``ON CONFLICT DO NOTHING`` on the unique ``batch_id`` means a concurrent + commit for the same batch never replaces the winner's receipt; the + service re-reads the stored receipt after commit and returns that one. + """ + try: + statement = ( + pg_insert(TelemetryBatchReceipt) + .values( + receipt_id=receipt.receipt_id, + batch_id=str(receipt.batch_id), + enrollment_id=receipt.enrollment_id, + research_session_id=receipt.research_session_id, + accepted_at=receipt.accepted_at, + receipt_json=receipt.ack.model_dump(mode="json"), + ) + .on_conflict_do_nothing(index_elements=["batch_id"]) + ) + self.session.execute(statement) + except SQLAlchemyError as error: + raise StoreUnavailable("receipt store is unavailable") from error + + def commit(self) -> None: + """Commit the whole batch (events + receipt) atomically.""" + try: + self.session.commit() + except IntegrityError as error: + self.session.rollback() + # A concurrent same-batch commit is handled with ON CONFLICT DO + # NOTHING, so an integrity error here is a genuine store failure + # (for example a globally reused event id): never a false accept. + raise StoreUnavailable( + "event insert hit a database integrity constraint" + ) from error + except SQLAlchemyError as error: + self.session.rollback() + raise StoreUnavailable("event store is unavailable") from error + + def rollback(self) -> None: + """Discard the current transaction.""" + self.session.rollback() + + def get_receipt(self, batch_id: uuid.UUID) -> Optional[BatchReceipt]: + """Fetch the immutable receipt for a batch, or ``None``.""" + statement = select(TelemetryBatchReceipt).where( + TelemetryBatchReceipt.batch_id == str(batch_id) + ) + row = self.session.execute(statement).scalars().first() + if row is None: + return None + from .models import TelemetryBatchAckV1 + + return BatchReceipt( + receipt_id=row.receipt_id, + batch_id=uuid.UUID(str(row.batch_id)), + enrollment_id=row.enrollment_id, + research_session_id=row.research_session_id, + accepted_at=row.accepted_at, + ack=TelemetryBatchAckV1.model_validate(row.receipt_json), + ) + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[uuid.UUID] + ) -> Optional[int]: + """Return the max stored sequence for an emitter, or ``None``. + + The cursor is a pure projection of the event store, so no cursor row is + persisted: ``max(emitter_sequence)`` for the ``(emitter, session)``. + """ + statement = select(func.max(ResearchEvent.emitter_sequence)).where( + ResearchEvent.emitter_id == emitter_id, + ResearchEvent.research_session_id == research_session_id, + ) + value = self.session.execute(statement).scalar() + return int(value) if value is not None else None + + +def get_receipt_by_id( + session: Session, receipt_id: uuid.UUID +) -> Optional[BatchReceipt]: + """Fetch an immutable receipt by its receipt id, or ``None``.""" + statement = select(TelemetryBatchReceipt).where( + TelemetryBatchReceipt.receipt_id == receipt_id + ) + row = session.execute(statement).scalars().first() + if row is None: + return None + from .models import TelemetryBatchAckV1 + + return BatchReceipt( + receipt_id=row.receipt_id, + batch_id=uuid.UUID(str(row.batch_id)), + enrollment_id=row.enrollment_id, + research_session_id=row.research_session_id, + accepted_at=row.accepted_at, + ack=TelemetryBatchAckV1.model_validate(row.receipt_json), + ) diff --git a/src/research/telemetry/ingestion/validation.py b/src/research/telemetry/ingestion/validation.py new file mode 100644 index 0000000..e9899d3 --- /dev/null +++ b/src/research/telemetry/ingestion/validation.py @@ -0,0 +1,137 @@ +"""Per-event validation for ingestion batches. + +Validation never repairs or reclassifies an event. It returns typed +:class:`~research.telemetry.ingestion.models.IngestionIssue` findings: schema/version/ +provenance/context problems and a defense-in-depth sensitive-field scan. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from research.telemetry.validation import ( + TelemetryValidationCode, + validate_canonical_event, +) + +from .enums import IngestionReasonCode +from .models import IngestionContext, IngestionIssue + +if TYPE_CHECKING: + from research.telemetry.models import CanonicalEventV1 + +__all__ = ["validate_batch_event"] + +_SCHEMA_VERSION = "1" + +_CODE_MAP: dict[TelemetryValidationCode, IngestionReasonCode] = { + TelemetryValidationCode.SCHEMA_INVALID: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.SCHEMA_VERSION_UNSUPPORTED: ( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION + ), + TelemetryValidationCode.SCHEMA_VERSION_MISSING: ( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION + ), + TelemetryValidationCode.MISSING_EVENT_ID: IngestionReasonCode.MISSING_EVENT_ID, + TelemetryValidationCode.MISSING_PROVENANCE: IngestionReasonCode.MISSING_PROVENANCE, + TelemetryValidationCode.MISSING_EMITTER_ID: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.INVALID_EMITTER_SEQUENCE: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.MISSING_OCCURRED_AT: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.SECRET_PRESENT: IngestionReasonCode.SENSITIVE_PAYLOAD, + TelemetryValidationCode.PRIVACY_BLOCKED: IngestionReasonCode.PRIVACY_BLOCKED, +} + + +def _issue( + code: IngestionReasonCode, field: str, message: str +) -> IngestionIssue: + return IngestionIssue(code=code, field=field, message=message, permanent=True) + + +def validate_batch_event( + event: CanonicalEventV1, + context: Optional[IngestionContext], + *, + telemetry_schema_version: str = _SCHEMA_VERSION, +) -> list[IngestionIssue]: + """Return every permanent reason ``event`` cannot be stored.""" + issues: list[IngestionIssue] = [] + + if event.schema_version != telemetry_schema_version: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + "schema_version", + ( + f"event schema_version {event.schema_version!r} does not match " + f"the batch telemetry_schema_version {telemetry_schema_version!r}" + ), + ) + ) + elif event.schema_version != _SCHEMA_VERSION: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + "schema_version", + f"unsupported telemetry schema version {event.schema_version!r}", + ) + ) + + for finding in validate_canonical_event(event): + mapped = _CODE_MAP.get(finding.code) + if mapped is None: + # ``needs_review`` findings (unknown type/source) are preserved, not + # rejected; the stored record retains the unknown token and coverage. + continue + issues.append(_issue(mapped, finding.field, finding.message)) + + if not event.provenance: + issues.append( + _issue( + IngestionReasonCode.MISSING_PROVENANCE, + "provenance", + "provenance is required to persist a canonical event", + ) + ) + + if context is not None: + if event.study_id is not None and event.study_id != context.study_id: + issues.append( + _issue( + IngestionReasonCode.CONTEXT_MISMATCH, + "study_id", + "event study does not match the authorized study", + ) + ) + if event.revision_id is not None and event.revision_id != context.study_revision_id: + issues.append( + _issue( + IngestionReasonCode.CONTEXT_MISMATCH, + "revision_id", + "event revision does not match the authorized revision", + ) + ) + if ( + event.enrollment_id is not None + and event.enrollment_id != context.enrollment_id + ): + issues.append( + _issue( + IngestionReasonCode.CONTEXT_MISMATCH, + "enrollment_id", + "event enrollment does not match the authorized enrollment", + ) + ) + if ( + event.research_session_id is not None + and event.research_session_id != context.research_session_id + ): + issues.append( + _issue( + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + "research_session_id", + "event belongs to a different research session", + ) + ) + + return issues diff --git a/src/research/telemetry/models.py b/src/research/telemetry/models.py new file mode 100644 index 0000000..d9f4cc8 --- /dev/null +++ b/src/research/telemetry/models.py @@ -0,0 +1,146 @@ +"""Pydantic v2 contracts for the canonical telemetry envelope (``CanonicalEventV1``). + +Design rules encoded here: + +* An event is immutable after construction (``frozen=True``). +* ``usage_tokens`` is nullable and is paired with an explicit + :class:`Coverage` ``usage_capability``: ``0`` means an observed zero and + ``None`` means "not exposed". The two are never conflated. +* Unknown enum/input values are preserved on ``unknown_*`` fields and flagged + with a ``NEEDS_REVIEW`` coverage state rather than reclassified. +* ``SECRET`` material is never persistable; the privacy engine removes it + before serialization. The model itself carries only sanitized payload. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import CanonicalFidelity, CoverageState + +_FROZEN = ConfigDict(extra="forbid", frozen=True) + + +class Provenance(BaseModel): + """Where a canonical event came from and how faithfully it was derived.""" + + model_config = _FROZEN + + source: str + source_event_id: Optional[str] = None + normalizer_version: str + adapter_version: Optional[str] = None + fidelity: CanonicalFidelity = CanonicalFidelity.NORMALIZED + evidence_digest: Optional[str] = None + + +class PrivacySummary(BaseModel): + """What the privacy filter did to one event.""" + + model_config = _FROZEN + + policy_digest: Optional[str] = None + actions: dict[str, int] = Field(default_factory=dict) + redacted_fields: list[str] = Field(default_factory=list) + blocked: bool = False + block_reason: Optional[str] = None + field_classes: dict[str, int] = Field(default_factory=dict) + + +class Coverage(BaseModel): + """Coverage of one measured/observed field or capability.""" + + model_config = _FROZEN + + state: CoverageState = CoverageState.UNKNOWN + reason: Optional[str] = None + capability: Optional[str] = None + + +class EventMetrics(BaseModel): + """Numeric measurements attached to an event. + + ``usage_tokens`` is ``Optional`` and paired with ``usage_capability``: + an observed zero is ``0`` with ``AVAILABLE`` coverage, while an unexposed + count is ``None`` with ``UNAVAILABLE`` coverage. + """ + + model_config = _FROZEN + + usage_tokens: Optional[int] = None + usage_capability: Coverage = Field(default_factory=Coverage) + latency_ms: Optional[int] = None + counts: dict[str, int] = Field(default_factory=dict) + + +class Correlations(BaseModel): + """Opaque correlation handles that tie related events together.""" + + model_config = _FROZEN + + turn_id: Optional[str] = None + tool_call_id: Optional[str] = None + permission_id: Optional[str] = None + edit_id: Optional[str] = None + correlation_id: Optional[str] = None + + +class CanonicalEventV1(BaseModel): + """One immutable, privacy-filtered canonical telemetry event.""" + + model_config = _FROZEN + + event_id: UUID + # Required: the envelope version is never silently defaulted on ingestion. + # A missing version is a typed schema error, not an implicit "1". + schema_version: str + # Canonical dotted event type, or "unknown_source_event" when the source + # construct is not documented. The original value is kept in + # ``unknown_event_type``. + event_type: str + source: str + + study_id: Optional[UUID] = None + revision_id: Optional[UUID] = None + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + agent_run_id: Optional[str] = None + + occurred_at: datetime + # Per-emitter monotonic clock reading, if the emitter provided one. Global + # ordering is a query projection, never a stored fact. + monotonic_ns: Optional[int] = None + + emitter_id: str + emitter_sequence: int + + correlations: Correlations = Field(default_factory=Correlations) + lifecycle_state: Optional[str] = None + + # Sanitized, privacy-filtered mapping. Unknown fields are prefixed + # ``unknown_`` and never reclassified into a known concept. + payload: dict[str, Any] = Field(default_factory=dict) + + metrics: EventMetrics = Field(default_factory=EventMetrics) + privacy: PrivacySummary = Field(default_factory=PrivacySummary) + provenance: Provenance + coverage: Coverage = Field(default_factory=Coverage) + + # Preserved, un-reclassified inputs. + unknown_event_type: Optional[str] = None + unknown_source: Optional[str] = None + unknown_lifecycle_state: Optional[str] = None + + @property + def needs_review(self) -> bool: + """Whether any part of this event was not recognized and needs review.""" + return ( + self.coverage.state == CoverageState.NEEDS_REVIEW + or self.unknown_event_type is not None + or self.unknown_source is not None + or self.unknown_lifecycle_state is not None + ) diff --git a/src/research/telemetry/normalization/__init__.py b/src/research/telemetry/normalization/__init__.py new file mode 100644 index 0000000..e3e17b3 --- /dev/null +++ b/src/research/telemetry/normalization/__init__.py @@ -0,0 +1,81 @@ +"""Source normalizers that map observations into canonical candidates. + +* :mod:`research.telemetry.normalization.generic_acp` - the built-in ACP + normalizer (plus a sanitized ``unknown_source_event`` fallback and the optional + :class:`~research.telemetry.normalization.generic_acp.AgentAdapter` enrichment + interface). + +Normalizers are pure: :func:`materialize_candidate` attaches the event id and +per-emitter sequence through the :class:`~research.telemetry.builder.EventBuilder`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from .generic_acp import ( + GENERIC_ACP_NORMALIZER_VERSION, + AgentAdapter, + GenericAcpNormalizer, + enrich_with_adapter, +) +from .models import CanonicalCandidateV1, NormalizationResultV1 + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from ..builder import EventBuilder + from ..models import CanonicalEventV1 + +__all__ = [ + "GENERIC_ACP_NORMALIZER_VERSION", + "AgentAdapter", + "CanonicalCandidateV1", + "GenericAcpNormalizer", + "NormalizationResultV1", + "enrich_with_adapter", + "materialize_candidate", +] + + +def materialize_candidate( + builder: EventBuilder, + candidate: CanonicalCandidateV1, + result: NormalizationResultV1, + *, + emitter_id: str, + occurred_at: datetime, + study_id: Optional[UUID] = None, + revision_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + research_session_id: Optional[UUID] = None, + agent_run_id: Optional[str] = None, + monotonic_ns: Optional[int] = None, + event_id: Optional[UUID] = None, + emitter_sequence: Optional[int] = None, +) -> CanonicalEventV1: + """Build an immutable canonical event from a candidate and its result.""" + return builder.build( + emitter_id=emitter_id, + event_type=candidate.event_type, + source=result.source, + occurred_at=occurred_at, + normalizer_version=result.normalizer_version, + payload=candidate.payload, + fidelity=candidate.fidelity, + adapter_version=candidate.adapter_version or result.adapter_version, + source_event_id=result.source_event_id, + metrics=candidate.metrics, + coverage=candidate.coverage, + lifecycle_state=candidate.lifecycle_state, + correlations=candidate.correlations, + monotonic_ns=monotonic_ns, + study_id=study_id, + revision_id=revision_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + agent_run_id=agent_run_id, + event_id=event_id, + emitter_sequence=emitter_sequence, + ) diff --git a/src/research/telemetry/normalization/generic_acp.py b/src/research/telemetry/normalization/generic_acp.py new file mode 100644 index 0000000..4a94f82 --- /dev/null +++ b/src/research/telemetry/normalization/generic_acp.py @@ -0,0 +1,653 @@ +"""Built-in ACP source normalizer: observations into canonical concepts. + +This module hosts the built-in, pure ACP normalizer: + +* :class:`GenericAcpNormalizer` - ACP JSON-RPC messages. It maps **only + documented, stable constructs**. It is not a copy of the ACP method + vocabulary: a method/notification becomes one of the canonical + :class:`~research.telemetry.enums.CanonicalEventType` concepts, and an unknown + method becomes ``unknown_source_event`` carrying sanitized metadata (method + name and parameter *names*, never values) with a ``NEEDS_REVIEW`` coverage + state. + +Vendor-specific semantic enrichment is intentionally out of scope for the ACP +normalizer; an :class:`AgentAdapter` may enrich a candidate later without +deleting or replacing the generic observation. IDE canonicalization belongs to +the IntelliJ plugin, not the server. + +Correlation note: ACP is a request/response protocol whose updates arrive as +notifications. Two canonical facts therefore need *stream* context, not a +single frame: + +* ``permission.decided`` is derived from the **host response** to a previously + observed ``session/request_permission`` (the response only carries the + selected option id, so the request's options are needed to resolve the option + *kind*); +* ``agent.message.completed`` is derived from the response to a previously + observed ``session/prompt`` request. + +Those correlations (and the "``tool.created`` once per tool call" rule) are held +as bounded, per-session state on the normalizer instance. Every other mapping is +a pure function of one message. The state never changes an observation's +identity, order, or fidelity. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Optional, Protocol + +from ..enums import CanonicalEventType, CanonicalFidelity, CoverageState, EventSource +from ..models import Correlations, Coverage, EventMetrics +from .models import CanonicalCandidateV1, NormalizationResultV1 + +GENERIC_ACP_NORMALIZER_VERSION = "generic-acp-v1" + +#: Canonical ACP permission option kinds that mean "allow". +_ALLOW_OPTION_KINDS = frozenset({"allow_once", "allow_always"}) +#: Canonical ACP permission option kinds that mean "reject". +_REJECT_OPTION_KINDS = frozenset({"reject_once", "reject_always"}) + +_METHOD_RULES: dict[str, tuple[str, CanonicalEventType]] = { + "initialize": ("acp.initialize", CanonicalEventType.INTERACTION_STARTED), + "session/new": ("acp.session.new", CanonicalEventType.INTERACTION_STARTED), + "session/load": ("acp.session.load", CanonicalEventType.INTERACTION_STARTED), + "session/prompt": ("acp.session.prompt", CanonicalEventType.AGENT_MESSAGE_STARTED), + "session/cancel": ("acp.session.cancel", CanonicalEventType.INTERACTION_COMPLETED), + "fs/read_text_file": ("acp.fs.read_text_file", CanonicalEventType.IDE_FILE_OPENED), + "fs/write_text_file": ("acp.fs.write_text_file", CanonicalEventType.IDE_FILE_SAVED), + "terminal/create": ("acp.terminal.create", CanonicalEventType.TOOL_STARTED), +} + + +def _candidate( + event_type: CanonicalEventType, + rule_id: str, + *, + payload: Optional[dict[str, Any]] = None, + lifecycle_state: Optional[str] = None, + coverage: Optional[Coverage] = None, + correlations: Optional[Correlations] = None, + metrics: Optional[EventMetrics] = None, +) -> CanonicalCandidateV1: + return CanonicalCandidateV1( + event_type=event_type, + fidelity=CanonicalFidelity.NORMALIZED, + mapping_rule_id=rule_id, + payload=payload or {}, + correlations=correlations or Correlations(), + lifecycle_state=lifecycle_state, + coverage=coverage or Coverage(state=CoverageState.AVAILABLE), + metrics=metrics or EventMetrics(), + ) + + +class AgentAdapter(Protocol): + """Optional vendor adapter that enriches (never replaces) generic events.""" + + adapter_version: str + mapping_rule_version: str + supported_release_ranges: list[str] + + def enrich( + self, candidate: CanonicalCandidateV1 + ) -> Optional[CanonicalCandidateV1]: # pragma: no cover - structural protocol + ... + + +def enrich_with_adapter( + candidate: CanonicalCandidateV1, adapter: AgentAdapter +) -> CanonicalCandidateV1: + """Apply an adapter without deleting or replacing generic fields. + + The generic candidate's payload wins on conflict, so an adapter can only add + fields; it can never erase or rewrite the source observation. + """ + enriched = adapter.enrich(candidate) + if enriched is None: + return candidate + merged_payload = {**enriched.payload, **candidate.payload} + return candidate.model_copy( + update={"payload": merged_payload, "adapter_version": adapter.adapter_version} + ) + + +def _unknown_candidate( + method: Optional[str], params: Any, *, source_event_id: Optional[str] = None +) -> CanonicalCandidateV1: + payload: dict[str, Any] = {} + if method is not None: + payload["unknown_method"] = method + if isinstance(params, Mapping): + # Parameter *names* only; values may be sensitive. + payload["param_names"] = sorted(str(key) for key in params) + if source_event_id is not None: + payload["source_event_id"] = source_event_id + return _candidate( + CanonicalEventType.UNKNOWN_SOURCE_EVENT, + "acp.unknown", + payload=payload, + coverage=Coverage( + state=CoverageState.NEEDS_REVIEW, + reason="unrecognized ACP construct preserved as metadata only", + ), + ) + + +def _tool_payload(location: Mapping[str, Any]) -> dict[str, Any]: + payload: dict[str, Any] = {} + for source_key, target_key in ( + ("toolCallId", "tool_call_id"), + ("title", "tool_name"), + ("kind", "tool_kind"), + ("status", "status"), + ): + value = location.get(source_key) + if value is not None: + payload[target_key] = value + raw_input = location.get("rawInput") + if raw_input is not None: + payload["arguments"] = raw_input + content = location.get("content") + if content is not None: + payload["content"] = content + return payload + + +def _tool_call_id(update: Mapping[str, Any]) -> Optional[str]: + value = update.get("toolCallId") + return value if isinstance(value, str) and value else None + + +def _status(update: Mapping[str, Any]) -> str: + return str(update.get("status") or "").lower() + + +def _terminal_tool_candidate( + update: Mapping[str, Any], +) -> Optional[CanonicalCandidateV1]: + """Map a terminal ``tool_call``/``tool_call_update`` status; else ``None``.""" + status = _status(update) + if status in {"completed", "success", "succeeded"}: + return _candidate( + CanonicalEventType.TOOL_COMPLETED, + "acp.update.tool_call_update.completed", + payload=_tool_payload(update), + lifecycle_state="completed", + ) + if status in {"failed", "error"}: + return _candidate( + CanonicalEventType.TOOL_FAILED, + "acp.update.tool_call_update.failed", + payload=_tool_payload(update), + lifecycle_state="failed", + ) + return None + + +def _first_int(source: Mapping[str, Any], keys: tuple[str, ...]) -> Optional[int]: + for key in keys: + value = source.get(key) + if isinstance(value, bool): + continue + if isinstance(value, int): + return value + return None + + +def _usage_metrics(source: Any) -> EventMetrics: + """Build usage metrics from an agent-reported usage mapping. + + A reported total (or a derivable input+output total) is an observed value + with ``AVAILABLE`` coverage - including an observed ``0``. When the agent + reports no token counts at all, the value stays ``None`` with ``UNAVAILABLE`` + coverage; a missing measurement is never coerced to zero. + """ + tokens: Optional[int] = None + if isinstance(source, Mapping): + tokens = _first_int(source, ("totalTokens", "total_tokens")) + if tokens is None: + input_tokens = _first_int(source, ("inputTokens", "input_tokens")) + output_tokens = _first_int(source, ("outputTokens", "output_tokens")) + if input_tokens is not None or output_tokens is not None: + tokens = (input_tokens or 0) + (output_tokens or 0) + if tokens is None: + tokens = _first_int(source, ("used",)) + if tokens is None: + return EventMetrics( + usage_tokens=None, + usage_capability=Coverage( + state=CoverageState.UNAVAILABLE, + reason="agent did not report token usage", + capability="usage", + ), + ) + return EventMetrics( + usage_tokens=tokens, + usage_capability=Coverage(state=CoverageState.AVAILABLE, capability="usage"), + ) + + +def _decision_for(option_id: str, kind: Optional[str]) -> str: + """Resolve an allow/reject/cancelled decision from a selected option.""" + if kind in _ALLOW_OPTION_KINDS: + return "allow" + if kind in _REJECT_OPTION_KINDS: + return "reject" + text = option_id.lower() + if "reject" in text or "deny" in text: + return "reject" + if "allow" in text or "approve" in text or "accept" in text: + return "allow" + return "selected" + + +def _normalized_options( + params: Any, +) -> tuple[list[dict[str, str]], dict[str, Optional[str]]]: + """Return ``(payload_options, {option_id: kind})`` from request params.""" + payload_options: list[dict[str, str]] = [] + option_kinds: dict[str, Optional[str]] = {} + if not isinstance(params, Mapping): + return payload_options, option_kinds + raw_options = params.get("options") + if not isinstance(raw_options, list): + return payload_options, option_kinds + for option in raw_options: + if not isinstance(option, Mapping): + continue + option_id = option.get("optionId") + if not isinstance(option_id, str): + option_id = option.get("option_id") + kind = option.get("kind") + entry: dict[str, str] = {} + if isinstance(option_id, str) and option_id: + entry["option_id"] = option_id + option_kinds[option_id] = kind if isinstance(kind, str) else None + if isinstance(kind, str): + entry["kind"] = kind + if entry: + payload_options.append(entry) + return payload_options, option_kinds + + +def _tool_call_request_tool_id(params: Any) -> Optional[str]: + if not isinstance(params, Mapping): + return None + tool_call = params.get("toolCall") + if not isinstance(tool_call, Mapping): + return None + value = tool_call.get("toolCallId") + return value if isinstance(value, str) and value else None + + +class GenericAcpNormalizer: + """Maps ACP JSON-RPC messages to canonical candidates. + + A normalizer instance is bound to one observer stream (session): it keeps + only the correlation state required for ``permission.decided`` / + ``agent.message.completed`` and for emitting ``tool.created`` exactly once + per tool call. + """ + + normalizer_version = GENERIC_ACP_NORMALIZER_VERSION + + def __init__(self) -> None: + #: jsonrpc permission request id -> {"options": {optionId: kind}, "tool_call_id": ...} + self._pending_permissions: dict[str, dict[str, Any]] = {} + #: jsonrpc ``session/prompt`` request ids awaiting their response. + self._pending_prompts: set[str] = set() + #: tool call ids for which ``tool.created`` was already emitted. + self._created_tool_calls: set[str] = set() + + def normalize( + self, + message: Any, + *, + source_event_id: Optional[str] = None, + direction: Optional[str] = None, + ) -> NormalizationResultV1: + """Normalize one ACP message (zero or more candidates). + + ``direction`` is the optional ``host_to_agent`` / ``agent_to_host`` frame + direction. When provided it disambiguates responses (a permission + decision is only taken from a host response; a message completion only + from an agent response); when omitted, correlation falls back to the + JSON-RPC id alone. + """ + if not isinstance(message, Mapping): + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=source_event_id, + candidates=[], + unmapped_reason="message is not a JSON object", + ) + + method = message.get("method") + params = message.get("params") + message_id = message.get("id") + resolved_source_id = ( + source_event_id + if source_event_id is not None + else (str(message_id) if message_id is not None else None) + ) + + if method == "session/update" and isinstance(params, Mapping): + update = params.get("update") + if isinstance(update, Mapping): + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=self._update_candidates(update), + ) + + if method == "session/request_permission": + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[self._permission_request(message_id, params)], + ) + + if method == "session/prompt" and message_id is not None: + self._pending_prompts.add(str(message_id)) + + if isinstance(method, str) and method in _METHOD_RULES: + rule_id, event_type = _METHOD_RULES[method] + payload: dict[str, Any] = {} + if isinstance(params, Mapping): + session_id = params.get("sessionId") + if isinstance(session_id, str): + payload["session_id"] = session_id + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[_candidate(event_type, rule_id, payload=payload)], + ) + + if "error" in message: + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[self._agent_error(message, direction)], + ) + + # A response carries no method; pair it with an observed request. + if method is None and message_id is not None: + response = self._response_candidate(str(message_id), message, direction) + if response is not None: + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[response], + ) + + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[_unknown_candidate(method, params, source_event_id=resolved_source_id)], + ) + + # -- session/update ---------------------------------------------------- + + def _update_candidates( + self, update: Mapping[str, Any] + ) -> list[CanonicalCandidateV1]: + kind = update.get("sessionUpdate") + + if kind == "agent_message_chunk": + payload: dict[str, Any] = {"message_kind": "assistant"} + if isinstance(update.get("messageId"), str): + payload["message_id"] = update["messageId"] + if update.get("content") is not None: + payload["content"] = update["content"] + return [ + _candidate( + CanonicalEventType.AGENT_MESSAGE_STARTED, + "acp.update.agent_message_chunk", + payload=payload, + lifecycle_state="started", + ) + ] + + if kind == "agent_thought_chunk": + payload = {"message_kind": "thought"} + if isinstance(update.get("messageId"), str): + payload["message_id"] = update["messageId"] + if update.get("content") is not None: + # Reasoning is CONTENT and prohibited by default; it is only kept + # when content capture is explicitly allowed AND consented. + payload["reasoning"] = update["content"] + return [ + _candidate( + CanonicalEventType.AGENT_MESSAGE_STARTED, + "acp.update.agent_thought_chunk", + payload=payload, + lifecycle_state="started", + ) + ] + + if kind == "tool_call": + return self._tool_call_candidates(update) + + if kind == "tool_call_update": + terminal = _terminal_tool_candidate(update) + if terminal is not None: + return [terminal] + return [ + _candidate( + CanonicalEventType.TOOL_STARTED, + "acp.update.tool_call_update", + payload=_tool_payload(update), + lifecycle_state="started", + ) + ] + + if kind == "plan": + return [self._plan_candidate(update)] + + if kind == "usage_update": + return [ + _candidate( + CanonicalEventType.USAGE_UPDATED, + "acp.update.usage_update", + metrics=_usage_metrics(update), + ) + ] + + return [ + _unknown_candidate( + f"session/update:{kind}" if kind else "session/update", update + ) + ] + + def _tool_call_candidates( + self, update: Mapping[str, Any] + ) -> list[CanonicalCandidateV1]: + tool_call_id = _tool_call_id(update) + already_created = ( + tool_call_id is not None and tool_call_id in self._created_tool_calls + ) + if already_created: + candidates = [ + _candidate( + CanonicalEventType.TOOL_STARTED, + "acp.update.tool_call", + payload=_tool_payload(update), + lifecycle_state="started", + ) + ] + else: + if tool_call_id is not None: + self._created_tool_calls.add(tool_call_id) + candidates = [ + _candidate( + CanonicalEventType.TOOL_CREATED, + "acp.update.tool_call.created", + payload=_tool_payload(update), + lifecycle_state="pending", + ) + ] + # A single-shot ``tool_call`` can already be terminal. + terminal = _terminal_tool_candidate(update) + if terminal is not None: + candidates.append(terminal) + return candidates + + def _plan_candidate(self, update: Mapping[str, Any]) -> CanonicalCandidateV1: + payload: dict[str, Any] = {} + entries = update.get("entries") + if isinstance(entries, list): + payload["plan_size"] = len(entries) + status_counts: dict[str, int] = {} + for entry in entries: + if not isinstance(entry, Mapping): + continue + status = entry.get("status") + if isinstance(status, str): + status_counts[status] = status_counts.get(status, 0) + 1 + if status_counts: + # A list of {"status", "count"} records, not a status-keyed map: + # the fail-closed classifier treats status strings used as keys + # as unclassified content and would redact their values. + payload["plan_status_counts"] = [ + {"status": status, "count": count} + for status, count in sorted(status_counts.items()) + ] + return _candidate( + CanonicalEventType.PLAN_UPDATED, "acp.update.plan", payload=payload + ) + + # -- permissions ------------------------------------------------------- + + def _permission_request( + self, message_id: Any, params: Any + ) -> CanonicalCandidateV1: + permission_id = str(message_id) if message_id is not None else None + tool_call_id = _tool_call_request_tool_id(params) + options, option_kinds = _normalized_options(params) + payload: dict[str, Any] = {} + if tool_call_id is not None: + payload["tool_call_id"] = tool_call_id + if options: + payload["options"] = options + payload["option_count"] = len(options) + if permission_id is not None: + self._pending_permissions[permission_id] = { + "tool_call_id": tool_call_id, + "options": option_kinds, + } + return _candidate( + CanonicalEventType.PERMISSION_REQUESTED, + "acp.permission.request", + payload=payload, + correlations=Correlations( + permission_id=permission_id, tool_call_id=tool_call_id + ), + ) + + def _permission_decided( + self, permission_id: str, message: Mapping[str, Any] + ) -> CanonicalCandidateV1: + spec = self._pending_permissions.pop(permission_id, {}) + tool_call_id = spec.get("tool_call_id") + option_kinds: Mapping[str, Optional[str]] = spec.get("options") or {} + result = message.get("result") + payload: dict[str, Any] = {} + if isinstance(result, Mapping): + outcome = result.get("outcome") + if isinstance(outcome, Mapping): + selected = outcome.get("outcome") + if isinstance(selected, str): + payload["outcome"] = selected + option_id = outcome.get("optionId") + if not isinstance(option_id, str): + option_id = outcome.get("option_id") + if selected == "selected" and isinstance(option_id, str): + kind = option_kinds.get(option_id) + payload["selected_option_id"] = option_id + if isinstance(kind, str): + payload["selected_option_kind"] = kind + payload["decision"] = _decision_for(option_id, kind) + elif selected == "cancelled": + payload["decision"] = "cancelled" + elif isinstance(outcome, str): + # Some ACP revisions surface the outcome string directly. + payload["outcome"] = outcome + if outcome == "cancelled": + payload["decision"] = "cancelled" + payload.setdefault("outcome", "unknown") + payload.setdefault("decision", "unknown") + return _candidate( + CanonicalEventType.PERMISSION_DECIDED, + "acp.permission.decided", + payload=payload, + correlations=Correlations( + permission_id=permission_id, tool_call_id=tool_call_id + ), + ) + + # -- responses / messages / errors ------------------------------------- + + def _response_candidate( + self, response_id: str, message: Mapping[str, Any], direction: Optional[str] + ) -> Optional[CanonicalCandidateV1]: + if response_id in self._pending_permissions and direction in ( + None, + "host_to_agent", + ): + return self._permission_decided(response_id, message) + if response_id in self._pending_prompts and direction in (None, "agent_to_host"): + self._pending_prompts.discard(response_id) + return self._message_completed(message) + return None + + def _message_completed(self, message: Mapping[str, Any]) -> CanonicalCandidateV1: + result = message.get("result") + payload: dict[str, Any] = {} + usage_source: Any = result + if isinstance(result, Mapping): + stop_reason = result.get("stopReason") + if not isinstance(stop_reason, str): + stop_reason = result.get("stop_reason") + if isinstance(stop_reason, str): + payload["stop_reason"] = stop_reason + if isinstance(result.get("usage"), Mapping): + usage_source = result["usage"] + return _candidate( + CanonicalEventType.AGENT_MESSAGE_COMPLETED, + "acp.message.completed", + payload=payload, + lifecycle_state="completed", + metrics=_usage_metrics(usage_source), + ) + + def _agent_error( + self, message: Mapping[str, Any], direction: Optional[str] + ) -> CanonicalCandidateV1: + error = message.get("error") + # A JSON-RPC error response observed on the wire is normally the agent + # rejecting a host request; a host->agent error is a host rejection. + error_source = "host" if direction == "host_to_agent" else "agent" + payload: dict[str, Any] = {"error_source": error_source} + if isinstance(error, Mapping): + if error.get("code") is not None: + payload["error_code"] = error["code"] + if error.get("message") is not None: + payload["error_message"] = error["message"] + return _candidate( + CanonicalEventType.AGENT_ERROR, "acp.error_response", payload=payload + ) + + +__all__ = [ + "AgentAdapter", + "GENERIC_ACP_NORMALIZER_VERSION", + "GenericAcpNormalizer", + "enrich_with_adapter", +] diff --git a/src/research/telemetry/normalization/models.py b/src/research/telemetry/normalization/models.py new file mode 100644 index 0000000..a3bc930 --- /dev/null +++ b/src/research/telemetry/normalization/models.py @@ -0,0 +1,47 @@ +"""Shared normalization contracts. + +A normalizer emits zero or more :class:`CanonicalCandidateV1` objects wrapped in +a :class:`NormalizationResultV1`. Candidates are pre-build: the +:class:`~research.telemetry.builder.EventBuilder` allocates the immutable event +id and per-emitter sequence, so normalizers stay pure and deterministic. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from ..enums import CanonicalEventType, CanonicalFidelity, EventSource +from ..models import Correlations, Coverage, EventMetrics + +_BASE_CONFIG = ConfigDict(extra="forbid") + + +class CanonicalCandidateV1(BaseModel): + """A pre-build canonical candidate produced by a normalizer.""" + + model_config = _BASE_CONFIG + + event_type: CanonicalEventType + fidelity: CanonicalFidelity = CanonicalFidelity.NORMALIZED + mapping_rule_id: str + payload: dict[str, Any] = Field(default_factory=dict) + correlations: Correlations = Field(default_factory=Correlations) + lifecycle_state: Optional[str] = None + coverage: Coverage = Field(default_factory=Coverage) + metrics: EventMetrics = Field(default_factory=EventMetrics) + adapter_version: Optional[str] = None + + +class NormalizationResultV1(BaseModel): + """The result of normalizing one source observation.""" + + model_config = _BASE_CONFIG + + source: EventSource + normalizer_version: str + source_event_id: Optional[str] = None + adapter_version: Optional[str] = None + candidates: list[CanonicalCandidateV1] = Field(default_factory=list) + unmapped_reason: Optional[str] = None diff --git a/src/research/telemetry/privacy/__init__.py b/src/research/telemetry/privacy/__init__.py new file mode 100644 index 0000000..8055543 --- /dev/null +++ b/src/research/telemetry/privacy/__init__.py @@ -0,0 +1,50 @@ +"""Privacy filter for canonical telemetry (Issue 06). + +Public surface: + +* :mod:`research.telemetry.privacy.classify` - the field-classification + taxonomy and hard ``SECRET`` denylist. +* :mod:`research.telemetry.privacy.engine` - :class:`PrivacyPolicy`, + :func:`filter_event` and :func:`filter_payload`, which enforce data + minimization before any spool/log/retry/export. +""" + +from .classify import ( + BEHAVIORAL_TOKENS, + CODE_METADATA_TOKENS, + CONTENT_TOKENS, + SECRET_KEY_PATTERNS, + SECRET_VALUE_PATTERNS, + SYSTEM_TOKENS, + classify_event_payload, + classify_field, + contains_secret_value, + is_secret_key, + looks_secret_value, +) +from .engine import ( + REDACTED_MARKER, + FilterResult, + PrivacyPolicy, + filter_event, + filter_payload, +) + +__all__ = [ + "BEHAVIORAL_TOKENS", + "CODE_METADATA_TOKENS", + "CONTENT_TOKENS", + "FilterResult", + "PrivacyPolicy", + "REDACTED_MARKER", + "SECRET_KEY_PATTERNS", + "SECRET_VALUE_PATTERNS", + "SYSTEM_TOKENS", + "classify_event_payload", + "classify_field", + "contains_secret_value", + "filter_event", + "filter_payload", + "is_secret_key", + "looks_secret_value", +] diff --git a/src/research/telemetry/privacy/classify.py b/src/research/telemetry/privacy/classify.py new file mode 100644 index 0000000..dcb7ae7 --- /dev/null +++ b/src/research/telemetry/privacy/classify.py @@ -0,0 +1,250 @@ +"""Field classification for the canonical privacy filter. + +Classification is deterministic and fail-closed: + +1. A key matching the hard secret denylist, or a value matching a well-known + secret shape, is ``SECRET``. ``SECRET`` is never persistable regardless of + policy or consent. +2. Keys naming free content (prompts, reasoning, diffs, arguments, raw output) + are ``CONTENT``. +3. Keys naming file/code metadata are ``CODE_METADATA``. +4. Keys naming timing/process/schema facts are ``SYSTEM``. +5. Keys naming tool/permission/message lifecycle facts are ``BEHAVIORAL``. +6. Anything unrecognized is treated as ``CONTENT`` (fail closed): it is removed + unless content capture is explicitly allowed. Unrecognized *containers* are + traversed so their children can be classified individually. + +The classifier is vendor-independent and never invents a class from an ACP or +IDE method name. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ..enums import FieldClass + +__all__ = [ + "CONTENT_TOKENS", + "CODE_METADATA_TOKENS", + "SECRET_KEY_PATTERNS", + "SECRET_VALUE_PATTERNS", + "SYSTEM_TOKENS", + "BEHAVIORAL_TOKENS", + "classify_event_payload", + "classify_field", + "contains_secret_value", + "is_secret_key", + "looks_secret_value", +] + +# Keys that are always secret, matched on a normalized (lowercased, +# underscored) key. ``tokens`` (a count) deliberately does NOT match ``token``. +SECRET_KEY_PATTERNS: tuple[str, ...] = ( + "password", + "passwd", + "secret", + "api_key", + "apikey", + "authorization", + "bearer", + "credential", + "private_key", + "access_key", + "client_secret", + "cookie", + "session_token", + "auth_token", +) +_SECRET_KEY_EXACT = frozenset( + {"token", "auth", "authorization", "secret", "password", "credential"} +) + +SECRET_VALUE_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"), + re.compile(r"\bghp_[A-Za-z0-9]{8,}"), + re.compile(r"\bAKIA[0-9A-Z]{12,}"), + re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{4,}"), + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), +) + +CONTENT_TOKENS = frozenset( + { + "content", + "prompt", + "prompts", + "reasoning", + "thought", + "thoughts", + "completion", + "completions", + "response", + "output", + "stdout", + "stderr", + "diff", + "snippet", + "transcript", + "body", + "text", + "raw", + "payload", + "arguments", + "cot", + "chain", + "message_text", + } +) +CODE_METADATA_TOKENS = frozenset( + { + "path", + "file", + "filename", + "extension", + "language", + "line", + "lines", + "symbol", + "uri", + "directory", + "module", + "package", + "repo", + "workspace", + "document", + } +) +SYSTEM_TOKENS = frozenset( + { + "duration", + "latency", + "timestamp", + "clock", + "process", + "pid", + "version", + "schema", + "emitter", + "sequence", + "count", + "counts", + "size", + "bytes", + "host", + "os", + "arch", + "trace", + "span", + "level", + "tokens", + "tokens_used", + } +) +BEHAVIORAL_TOKENS = frozenset( + { + "tool", + "permission", + "turn", + "message", + "event", + "action", + "decision", + "status", + "state", + "role", + "model", + "result", + "outcome", + "error", + "name", + "id", + "type", + "call", + "edit", + "session", + "run", + "agent", + "capability", + "fidelity", + "kind", + "reason", + "method", + } +) + + +def _normalize_key(name: Any) -> str: + return re.sub(r"[^a-z0-9]+", "_", str(name).lower()).strip("_") + + +def _tokens(name: Any) -> set[str]: + return {token for token in re.split(r"[^a-z0-9]+", str(name).lower()) if token} + + +def is_secret_key(name: Any) -> bool: + """True when a mapping key names secret material.""" + normalized = _normalize_key(name) + if normalized in _SECRET_KEY_EXACT: + return True + if normalized.endswith(("_token", "_secret", "_password")): + return True + return any(pattern in normalized for pattern in SECRET_KEY_PATTERNS) + + +def looks_secret_value(value: Any) -> bool: + """True when a string value matches a well-known secret shape.""" + if not isinstance(value, str): + return False + return any(pattern.search(value) for pattern in SECRET_VALUE_PATTERNS) + + +def contains_secret_value(value: Any) -> bool: + """Recursively detect a secret-shaped value anywhere in ``value``.""" + if isinstance(value, Mapping): + return any( + is_secret_key(key) or contains_secret_value(item) + for key, item in value.items() + ) + if isinstance(value, (list, tuple)): + return any(contains_secret_value(item) for item in value) + return looks_secret_value(value) + + +def classify_field(name: Any, value: Any = None) -> FieldClass: + """Classify a single field by name (and value for secret shapes).""" + if is_secret_key(name) or looks_secret_value(value): + return FieldClass.SECRET + + tokens = _tokens(name) + if tokens & CONTENT_TOKENS: + return FieldClass.CONTENT + if tokens & CODE_METADATA_TOKENS: + return FieldClass.CODE_METADATA + if tokens & SYSTEM_TOKENS: + return FieldClass.SYSTEM + if tokens & BEHAVIORAL_TOKENS: + return FieldClass.BEHAVIORAL + + if isinstance(value, (Mapping, list, tuple)): + # Unrecognized containers are traversed; children are classified. + return FieldClass.SYSTEM + return FieldClass.CONTENT + + +def classify_event_payload(payload: Any, prefix: str = "") -> dict[str, FieldClass]: + """Flatten a sanitized payload into ``{dotted_path: FieldClass}``.""" + classified: dict[str, FieldClass] = {} + if isinstance(payload, Mapping): + for key, value in payload.items(): + path = f"{prefix}.{key}" if prefix else str(key) + classified[path] = classify_field(key, value) + if isinstance(value, (Mapping, list, tuple)): + classified.update(classify_event_payload(value, path)) + elif isinstance(payload, (list, tuple)): + for index, value in enumerate(payload): + path = f"{prefix}[{index}]" + classified[path] = classify_field(prefix, value) + if isinstance(value, (Mapping, list, tuple)): + classified.update(classify_event_payload(value, path)) + return classified diff --git a/src/research/telemetry/privacy/engine.py b/src/research/telemetry/privacy/engine.py new file mode 100644 index 0000000..6fbe54a --- /dev/null +++ b/src/research/telemetry/privacy/engine.py @@ -0,0 +1,307 @@ +"""Shared privacy policy engine: classification, redaction and hashing. + +Redaction happens here, before any spool, log, retry queue, analytics, or +export. The engine is defensive in depth: even if a hand-crafted policy allows +everything, ``SECRET`` fields are dropped and a final recursive scan blocks any +residual secret shape from surviving serialization. + +Policy actions: + +* ``ALLOW`` - keep the value (``SYSTEM`` / ``BEHAVIORAL`` by default). +* ``REDACT`` - replace the value with a marker (``CONTENT`` without allowance). +* ``HASH`` - replace code metadata with a SHA-256 of its canonical JSON. +* ``DROP`` - remove the field entirely (``SECRET``). +* ``BLOCK`` - reject the whole event (explicitly blocked class). +""" + +from __future__ import annotations + +from typing import Any, Mapping, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.study.protocol.enums import TelemetryFieldClass + +from ..enums import FieldClass, PolicyAction +from ..models import CanonicalEventV1, PrivacySummary +from .classify import classify_field, contains_secret_value, looks_secret_value + +REDACTED_MARKER = "[REDACTED]" + +_BASE_CONFIG = ConfigDict(extra="forbid") + + +class PrivacyPolicy(BaseModel): + """Resolved field policy for one revision + consent state. + + Derived from a revision ``TelemetryPolicy`` plus the participant's active + consent; ``SECRET`` is never in the allowed set and is enforced separately. + """ + + model_config = _BASE_CONFIG + + allowed_field_classes: list[FieldClass] = Field( + default_factory=lambda: [ + FieldClass.SYSTEM, + FieldClass.BEHAVIORAL, + FieldClass.CODE_METADATA, + ] + ) + # CONTENT is persisted only when explicitly allowed AND consent is active. + content_allowed: bool = False + consent_active: bool = False + policy_digest: Optional[str] = None + blocked_field_classes: list[FieldClass] = Field(default_factory=list) + # "hash" (default) or "allow" for CODE_METADATA. + code_metadata_mode: str = "hash" + + @classmethod + def default(cls) -> PrivacyPolicy: + """The deny-by-default policy: metadata only, no content capture.""" + return cls() + + @classmethod + def from_revision_policy( + cls, + telemetry_policy: Any, + *, + consent_active: bool, + code_metadata_mode: str = "hash", + ) -> PrivacyPolicy: + """Resolve a revision ``TelemetryPolicy`` + consent into a policy.""" + classes: set[FieldClass] = set() + content_allowed = False + raw_classes = getattr(telemetry_policy, "allowed_field_classes", []) or [] + for raw in raw_classes: + value = raw.value if isinstance(raw, TelemetryFieldClass) else str(raw).upper() + if value == TelemetryFieldClass.STRUCTURAL.value: + classes.update({FieldClass.SYSTEM, FieldClass.BEHAVIORAL}) + elif value == TelemetryFieldClass.METRICS.value: + classes.add(FieldClass.SYSTEM) + elif value == TelemetryFieldClass.DIAGNOSTICS.value: + classes.add(FieldClass.BEHAVIORAL) + elif value == TelemetryFieldClass.CONTENT.value: + content_allowed = True + classes.add(FieldClass.CONTENT) + + if not classes: + classes.update( + {FieldClass.SYSTEM, FieldClass.BEHAVIORAL, FieldClass.CODE_METADATA} + ) + + digest = canonical_hash( + { + "allowed": sorted(field_class.value for field_class in classes), + "content_allowed": content_allowed, + } + ) + return cls( + allowed_field_classes=sorted(classes, key=lambda item: item.value), + content_allowed=content_allowed, + consent_active=consent_active, + policy_digest=digest, + code_metadata_mode=code_metadata_mode, + ) + + def action_for(self, field_class: FieldClass) -> PolicyAction: + """Resolve the policy action for one field class.""" + if field_class in self.blocked_field_classes: + return PolicyAction.BLOCK + if field_class == FieldClass.SECRET: + return PolicyAction.DROP + if field_class == FieldClass.CONTENT: + if self.content_allowed and self.consent_active: + return PolicyAction.ALLOW + return PolicyAction.REDACT + if field_class == FieldClass.CODE_METADATA: + return ( + PolicyAction.HASH + if self.code_metadata_mode == "hash" + else PolicyAction.ALLOW + ) + if field_class in self.allowed_field_classes: + return PolicyAction.ALLOW + return PolicyAction.DROP + + +class FilterResult(BaseModel): + """A filtered event plus the summary of what the filter did.""" + + model_config = _BASE_CONFIG + + event: CanonicalEventV1 + summary: PrivacySummary + + +class _Blocked(Exception): + def __init__(self, path: str, field_class: FieldClass) -> None: + super().__init__(path) + self.path = path + self.field_class = field_class + + +_DROPPED = object() + + +def _hash_value(value: Any) -> str: + return "sha256:" + canonical_hash({"value": value}) + + +def _record( + actions: dict[str, int], classes: dict[str, int], action: PolicyAction, field_class: FieldClass +) -> None: + actions[action.value] = actions.get(action.value, 0) + 1 + classes[field_class.value] = classes.get(field_class.value, 0) + 1 + + +def _apply( + value: Any, + policy: PrivacyPolicy, + path: str, + actions: dict[str, int], + classes: dict[str, int], + redacted: list[str], +) -> Any: + if isinstance(value, Mapping): + filtered: dict[Any, Any] = {} + for key, item in value.items(): + child = f"{path}.{key}" if path else str(key) + field_class = classify_field(key, item) + action = policy.action_for(field_class) + _record(actions, classes, action, field_class) + if action == PolicyAction.BLOCK: + raise _Blocked(child, field_class) + if action == PolicyAction.DROP: + redacted.append(child) + continue + if action == PolicyAction.REDACT: + filtered[key] = REDACTED_MARKER + redacted.append(child) + continue + if action == PolicyAction.HASH: + filtered[key] = _hash_value(item) + redacted.append(child) + continue + filtered[key] = _apply(item, policy, child, actions, classes, redacted) + return filtered + + if isinstance(value, (list, tuple)): + items = [] + for index, item in enumerate(value): + child = f"{path}[{index}]" + result = _apply(item, policy, child, actions, classes, redacted) + if result is not _DROPPED: + items.append(result) + return items + + if looks_secret_value(value): + # Defense in depth for scalar list items with no classifying key. + _record(actions, classes, PolicyAction.DROP, FieldClass.SECRET) + redacted.append(path) + return _DROPPED + return value + + +def _auxiliary_secret_field(event: CanonicalEventV1) -> Optional[str]: + """Name of the first non-payload envelope field carrying a secret, or None. + + The privacy policy applies to the whole envelope: provenance, correlations, + lifecycle/unknown values and run identity are all scanned so a secret can + never ride along outside ``payload``. + """ + candidates: list[tuple[str, Any]] = [ + ("provenance", event.provenance.model_dump(mode="json")), + ("correlations", event.correlations.model_dump(mode="json")), + ("lifecycle_state", event.lifecycle_state), + ("unknown_event_type", event.unknown_event_type), + ("unknown_source", event.unknown_source), + ("unknown_lifecycle_state", event.unknown_lifecycle_state), + ("agent_run_id", event.agent_run_id), + ] + for name, value in candidates: + if value and contains_secret_value(value): + return name + return None + + +def filter_event(event: CanonicalEventV1, policy: PrivacyPolicy) -> FilterResult: + """Filter ``event`` under ``policy`` and return the event plus its summary.""" + actions: dict[str, int] = {} + classes: dict[str, int] = {} + redacted: list[str] = [] + blocked = False + block_reason: Optional[str] = None + + try: + filtered_payload: Any = _apply( + event.payload, policy, "", actions, classes, redacted + ) + except _Blocked as blocked_error: + blocked = True + block_reason = ( + f"field {blocked_error.path} classified " + f"{blocked_error.field_class.value} is blocked by policy" + ) + filtered_payload = {} + + if not blocked and contains_secret_value(filtered_payload): + blocked = True + block_reason = "residual secret detected after filtering" + filtered_payload = {} + + # Secrets are excluded from every envelope field, not only the payload: + # provenance, correlation ids, lifecycle/unknown values and run identity are + # scanned too and fail closed. A residual secret anywhere blocks the event. + if not blocked: + auxiliary = _auxiliary_secret_field(event) + if auxiliary is not None: + blocked = True + block_reason = f"residual secret detected in {auxiliary} after filtering" + filtered_payload = {} + + summary = PrivacySummary( + policy_digest=policy.policy_digest, + actions=actions, + redacted_fields=sorted(redacted), + blocked=blocked, + block_reason=block_reason, + field_classes=classes, + ) + filtered_event = event.model_copy( + update={"payload": dict(filtered_payload), "privacy": summary} + ) + return FilterResult(event=filtered_event, summary=summary) + + +def filter_payload( + payload: Mapping[str, Any], policy: PrivacyPolicy +) -> tuple[dict[str, Any], PrivacySummary]: + """Filter a bare payload mapping (convenience for tests and extractors).""" + actions: dict[str, int] = {} + classes: dict[str, int] = {} + redacted: list[str] = [] + blocked = False + block_reason: Optional[str] = None + try: + filtered: Any = _apply(payload, policy, "", actions, classes, redacted) + except _Blocked as blocked_error: + blocked = True + block_reason = ( + f"field {blocked_error.path} classified " + f"{blocked_error.field_class.value} is blocked by policy" + ) + filtered = {} + if not blocked and contains_secret_value(filtered): + blocked = True + block_reason = "residual secret detected after filtering" + filtered = {} + summary = PrivacySummary( + policy_digest=policy.policy_digest, + actions=actions, + redacted_fields=sorted(redacted), + blocked=blocked, + block_reason=block_reason, + field_classes=classes, + ) + return dict(filtered), summary diff --git a/src/research/telemetry/projections/__init__.py b/src/research/telemetry/projections/__init__.py new file mode 100644 index 0000000..4480021 --- /dev/null +++ b/src/research/telemetry/projections/__init__.py @@ -0,0 +1,38 @@ +"""Derived telemetry coverage and metric projections (Issue 09). + +Projections are versioned, derived views over immutable canonical events. They +never replace source events, and missing capability data is represented as +``value: null`` with an explicit coverage state rather than a zero measurement. +""" + +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + DerivedMetricV1, + FamilyCoverage, + SessionRunSummaryV1, + TelemetryCoverageV1, +) +from .service import ( + as_canonical_event, + coverage_by_family, + derive_event_count_metric, + derive_usage_metric, + event_family, + session_summary, +) + +__all__ = [ + "COVERAGE_VERSION", + "DERIVATION_VERSION", + "DerivedMetricV1", + "FamilyCoverage", + "SessionRunSummaryV1", + "TelemetryCoverageV1", + "as_canonical_event", + "coverage_by_family", + "derive_event_count_metric", + "derive_usage_metric", + "event_family", + "session_summary", +] diff --git a/src/research/telemetry/projections/models.py b/src/research/telemetry/projections/models.py new file mode 100644 index 0000000..2cc5b62 --- /dev/null +++ b/src/research/telemetry/projections/models.py @@ -0,0 +1,83 @@ +"""Pydantic v2 contracts for derived telemetry projections (Issue 09). + +Projections are derived, versioned views over immutable canonical events. A +missing measurement is represented as ``value: null`` plus an explicit coverage +state, never as a zero-valued measurement, and a projection never replaces the +source events. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + +COVERAGE_VERSION = "coverage-v1" +DERIVATION_VERSION = "derived-v1" + + +class FamilyCoverage(BaseModel): + """Explicit per-event-family coverage counts.""" + + model_config = _FROZEN + + family: str + total: int + available: int = 0 + unavailable: int = 0 + partial: int = 0 + unknown: int = 0 + needs_review: int = 0 + + +class TelemetryCoverageV1(BaseModel): + """Versioned coverage projection for one revision and population.""" + + model_config = _FROZEN + + revision_id: UUID + population: str + coverage_version: str = COVERAGE_VERSION + families: list[FamilyCoverage] = Field(default_factory=list) + denominators: dict[str, int] = Field(default_factory=dict) + computed_at: Optional[datetime] = None + + +class DerivedMetricV1(BaseModel): + """A versioned derived metric that preserves coverage semantics. + + ``value`` is ``None`` when the metric cannot be computed (no observed + denominator); that is a coverage statement, never a zero measurement. + """ + + model_config = _FROZEN + + metric_id: str + numerator: Optional[int] = None + denominator: Optional[int] = None + coverage_predicate: str + derivation_version: str = DERIVATION_VERSION + value: Optional[float] = None + coverage_state: CoverageState = CoverageState.UNKNOWN + source_event_count: int = 0 + + +class SessionRunSummaryV1(BaseModel): + """A per-session summary that retains source provenance.""" + + model_config = _FROZEN + + research_session_id: Optional[UUID] = None + event_count: int = 0 + family_counts: dict[str, int] = Field(default_factory=dict) + agent_run_ids: list[str] = Field(default_factory=list) + first_occurred_at: Optional[datetime] = None + last_occurred_at: Optional[datetime] = None + derivation_version: str = DERIVATION_VERSION diff --git a/src/research/telemetry/projections/service.py b/src/research/telemetry/projections/service.py new file mode 100644 index 0000000..dbd8027 --- /dev/null +++ b/src/research/telemetry/projections/service.py @@ -0,0 +1,197 @@ +"""Pure projection functions over canonical research events (Issue 09). + +Every projection is deterministic, carries a derivation version and population, +and reads only committed event facts. Missing capability data stays visible as +``value: None`` with an explicit coverage state; projections never replace or +mutate the source events. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - used in annotations only +from typing import TYPE_CHECKING, Iterable, Optional, Sequence + +from research.telemetry.enums import CoverageState +from research.telemetry.models import CanonicalEventV1 + +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + DerivedMetricV1, + FamilyCoverage, + SessionRunSummaryV1, + TelemetryCoverageV1, +) + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "as_canonical_event", + "coverage_by_family", + "derive_event_count_metric", + "derive_usage_metric", + "event_family", + "session_summary", +] + + +def as_canonical_event(record: object) -> CanonicalEventV1: + """Return a canonical event for a stored record or a canonical event. + + The stored record carries the complete canonical envelope, so this is a + direct re-validation of the authority rather than a lossy field-by-field + reconstruction. + """ + if isinstance(record, CanonicalEventV1): + return record + envelope = getattr(record, "envelope", None) + if isinstance(envelope, dict) and envelope: + return CanonicalEventV1.model_validate(envelope) + if isinstance(record, dict): + return CanonicalEventV1.model_validate(record) + raise TypeError(f"cannot read a canonical event from {type(record)!r}") + + +def event_family(event_type: str) -> str: + """Return the canonical event family of a dotted event type.""" + return event_type.split(".", 1)[0] if event_type else "unknown" + + +def coverage_by_family( + events: Iterable[object], + revision_id: UUID, + *, + population: str = "all_events", + computed_at: Optional[datetime] = None, +) -> TelemetryCoverageV1: + """Build the explicit per-family coverage projection for a revision.""" + counts: dict[str, dict[str, int]] = {} + total = 0 + for record in events: + event = as_canonical_event(record) + total += 1 + family = event_family(event.event_type) + bucket = counts.setdefault( + family, + dict.fromkeys(CoverageState, 0), + ) + bucket[event.coverage.state.value] = bucket.get(event.coverage.state.value, 0) + 1 + + families = [ + FamilyCoverage( + family=family, + total=sum(bucket.values()), + available=bucket[CoverageState.AVAILABLE.value], + unavailable=bucket[CoverageState.UNAVAILABLE.value], + partial=bucket[CoverageState.PARTIAL.value], + unknown=bucket[CoverageState.UNKNOWN.value], + needs_review=bucket[CoverageState.NEEDS_REVIEW.value], + ) + for family, bucket in sorted(counts.items()) + ] + return TelemetryCoverageV1( + revision_id=revision_id, + population=population, + coverage_version=COVERAGE_VERSION, + families=families, + denominators={"events": total, "families": len(families)}, + computed_at=computed_at, + ) + + +def session_summary( + events: Iterable[object], + research_session_id: Optional[UUID] = None, +) -> SessionRunSummaryV1: + """Summarize the events of one session (or of all provided events).""" + selected = [ + as_canonical_event(record) + for record in events + if research_session_id is None + or record.research_session_id == research_session_id + ] + family_counts: dict[str, int] = {} + run_ids: set[str] = set() + first: Optional[datetime] = None + last: Optional[datetime] = None + for event in selected: + family = event_family(event.event_type) + family_counts[family] = family_counts.get(family, 0) + 1 + if event.agent_run_id: + run_ids.add(event.agent_run_id) + if first is None or event.occurred_at < first: + first = event.occurred_at + if last is None or event.occurred_at > last: + last = event.occurred_at + + return SessionRunSummaryV1( + research_session_id=research_session_id, + event_count=len(selected), + family_counts=family_counts, + agent_run_ids=sorted(run_ids), + first_occurred_at=first, + last_occurred_at=last, + derivation_version=DERIVATION_VERSION, + ) + + +def derive_usage_metric( + events: Sequence[object], + *, + metric_id: str = "usage_tokens_per_event", + population: str = "all_events", +) -> DerivedMetricV1: + """Derive mean usage tokens per event with explicit coverage. + + Only events whose ``usage_capability`` is ``AVAILABLE`` and whose value is + observed contribute; if none did, the value is ``None`` with ``UNAVAILABLE`` + coverage — never ``0``. + """ + canonical = [as_canonical_event(record) for record in events] + observed = [ + event + for event in canonical + if event.metrics.usage_capability.state == CoverageState.AVAILABLE + and event.metrics.usage_tokens is not None + ] + numerator = sum(event.metrics.usage_tokens or 0 for event in observed) + denominator = len(observed) + value: Optional[float] = None + state = CoverageState.UNAVAILABLE + if denominator > 0: + value = numerator / denominator + state = CoverageState.AVAILABLE + return DerivedMetricV1( + metric_id=metric_id, + numerator=numerator if denominator > 0 else None, + denominator=denominator if denominator > 0 else None, + coverage_predicate=( + f"{population}: usage_capability.state == AVAILABLE " + "and usage_tokens is not None" + ), + derivation_version=DERIVATION_VERSION, + value=value, + coverage_state=state, + source_event_count=len(canonical), + ) + + +def derive_event_count_metric( + events: Sequence[object], + *, + metric_id: str = "event_count", + population: str = "all_events", +) -> DerivedMetricV1: + """Derive the total event count (an observed count, not an inference).""" + count = len(events) + return DerivedMetricV1( + metric_id=metric_id, + numerator=count, + denominator=count, + coverage_predicate=f"{population}: all stored events", + derivation_version=DERIVATION_VERSION, + value=float(count), + coverage_state=CoverageState.AVAILABLE, + source_event_count=count, + ) diff --git a/src/research/telemetry/schema/__init__.py b/src/research/telemetry/schema/__init__.py new file mode 100644 index 0000000..b505ea8 --- /dev/null +++ b/src/research/telemetry/schema/__init__.py @@ -0,0 +1,90 @@ +"""Language-neutral JSON Schema for the canonical telemetry envelope. + +``canonical_event_v1.json`` is **generated** from +:class:`research.telemetry.models.CanonicalEventV1`, which is the single source +of truth for the event schema; the closed vocabularies (event types, coverage +states, fidelity) come from :mod:`research.telemetry.enums`. It is the +cross-language contract consumed by generated clients. + +The generation is deterministic and is asserted by the test suite, so the +checked-in file can never silently drift from the Python model:: + + python scripts/dev/generate_canonical_event_schema.py + +Unknown ``source``/``lifecycle`` semantics are identical to the model: both +accept an unknown token verbatim (the original is preserved on +``unknown_source``/``unknown_lifecycle_state``), so the schema does not enum- +restrict them. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.telemetry.enums import CANONICAL_SCHEMA_VERSION, CanonicalEventType +from research.telemetry.models import CanonicalEventV1 + +SCHEMA_DIR = Path(__file__).resolve().parent +CANONICAL_EVENT_V1_SCHEMA_PATH = SCHEMA_DIR / "canonical_event_v1.json" + +_SCHEMA_ID = "https://code4me.invalid/research/schema/canonical_event_v1.json" +_SCHEMA_DESCRIPTION = ( + "Language-neutral canonical telemetry envelope (schema version " + f"{CANONICAL_SCHEMA_VERSION}). Canonical event types are vendor-independent " + "concepts; SECRET fields never appear because the privacy filter removes " + "them before serialization." +) + + +def build_canonical_event_v1_schema() -> dict[str, Any]: + """Build the canonical event JSON Schema from the Python model.""" + schema = CanonicalEventV1.model_json_schema(ref_template="#/$defs/{model}") + schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" + schema["$id"] = _SCHEMA_ID + schema["title"] = "CanonicalEventV1" + schema["description"] = _SCHEMA_DESCRIPTION + + properties = schema.setdefault("properties", {}) + # The envelope version is required and pinned; it is never defaulted. + properties["schema_version"] = {"const": CANONICAL_SCHEMA_VERSION} + # The builder guarantees a known canonical type (an unrecognized source + # construct becomes ``unknown_source_event``), so the wire vocabulary is + # closed even though the raw model field is a plain string. + properties["event_type"] = { + "type": "string", + "enum": [event_type.value for event_type in CanonicalEventType], + } + return schema + + +def serialize_canonical_event_v1_schema(schema: dict[str, Any]) -> str: + """Serialize the schema deterministically (stable for review and diffs).""" + return json.dumps(schema, indent=2, ensure_ascii=False) + "\n" + + +def write_canonical_event_v1_schema( + path: Path = CANONICAL_EVENT_V1_SCHEMA_PATH, +) -> Path: + """Regenerate the checked-in JSON Schema from the Python model.""" + path.write_text( + serialize_canonical_event_v1_schema(build_canonical_event_v1_schema()), + encoding="utf-8", + ) + return path + + +def load_canonical_event_v1_schema() -> dict[str, Any]: + """Load the canonical event JSON Schema.""" + return json.loads(CANONICAL_EVENT_V1_SCHEMA_PATH.read_text()) + + +__all__ = [ + "CANONICAL_EVENT_V1_SCHEMA_PATH", + "SCHEMA_DIR", + "build_canonical_event_v1_schema", + "load_canonical_event_v1_schema", + "serialize_canonical_event_v1_schema", + "write_canonical_event_v1_schema", +] diff --git a/src/research/telemetry/schema/canonical_event_v1.json b/src/research/telemetry/schema/canonical_event_v1.json new file mode 100644 index 0000000..5ba74b0 --- /dev/null +++ b/src/research/telemetry/schema/canonical_event_v1.json @@ -0,0 +1,503 @@ +{ + "$defs": { + "CanonicalFidelity": { + "description": "How faithfully a canonical field represents its source observation.\n\nLowercase by canonical-spec convention; distinct from the uppercase receipt\nfidelity vocabulary in Issue 01.", + "enum": [ + "exact", + "normalized", + "inferred" + ], + "title": "CanonicalFidelity", + "type": "string" + }, + "Correlations": { + "additionalProperties": false, + "description": "Opaque correlation handles that tie related events together.", + "properties": { + "turn_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Turn Id" + }, + "tool_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Call Id" + }, + "permission_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Permission Id" + }, + "edit_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Edit Id" + }, + "correlation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Correlation Id" + } + }, + "title": "Correlations", + "type": "object" + }, + "Coverage": { + "additionalProperties": false, + "description": "Coverage of one measured/observed field or capability.", + "properties": { + "state": { + "$ref": "#/$defs/CoverageState", + "default": "UNKNOWN" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "capability": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Capability" + } + }, + "title": "Coverage", + "type": "object" + }, + "CoverageState": { + "description": "Whether a measurement/observation is present, absent, or disputed.\n\n``UNAVAILABLE`` means \"the source does not expose it\" and ``UNKNOWN`` means\n\"not observed\"; neither is zero or false.", + "enum": [ + "AVAILABLE", + "UNAVAILABLE", + "UNKNOWN", + "PARTIAL", + "NEEDS_REVIEW" + ], + "title": "CoverageState", + "type": "string" + }, + "EventMetrics": { + "additionalProperties": false, + "description": "Numeric measurements attached to an event.\n\n``usage_tokens`` is ``Optional`` and paired with ``usage_capability``:\nan observed zero is ``0`` with ``AVAILABLE`` coverage, while an unexposed\ncount is ``None`` with ``UNAVAILABLE`` coverage.", + "properties": { + "usage_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Usage Tokens" + }, + "usage_capability": { + "$ref": "#/$defs/Coverage" + }, + "latency_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Latency Ms" + }, + "counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Counts", + "type": "object" + } + }, + "title": "EventMetrics", + "type": "object" + }, + "PrivacySummary": { + "additionalProperties": false, + "description": "What the privacy filter did to one event.", + "properties": { + "policy_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Policy Digest" + }, + "actions": { + "additionalProperties": { + "type": "integer" + }, + "title": "Actions", + "type": "object" + }, + "redacted_fields": { + "items": { + "type": "string" + }, + "title": "Redacted Fields", + "type": "array" + }, + "blocked": { + "default": false, + "title": "Blocked", + "type": "boolean" + }, + "block_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Block Reason" + }, + "field_classes": { + "additionalProperties": { + "type": "integer" + }, + "title": "Field Classes", + "type": "object" + } + }, + "title": "PrivacySummary", + "type": "object" + }, + "Provenance": { + "additionalProperties": false, + "description": "Where a canonical event came from and how faithfully it was derived.", + "properties": { + "source": { + "title": "Source", + "type": "string" + }, + "source_event_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Event Id" + }, + "normalizer_version": { + "title": "Normalizer Version", + "type": "string" + }, + "adapter_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Adapter Version" + }, + "fidelity": { + "$ref": "#/$defs/CanonicalFidelity", + "default": "normalized" + }, + "evidence_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Digest" + } + }, + "required": [ + "source", + "normalizer_version" + ], + "title": "Provenance", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Language-neutral canonical telemetry envelope (schema version 1). Canonical event types are vendor-independent concepts; SECRET fields never appear because the privacy filter removes them before serialization.", + "properties": { + "event_id": { + "format": "uuid", + "title": "Event Id", + "type": "string" + }, + "schema_version": { + "const": "1" + }, + "event_type": { + "type": "string", + "enum": [ + "interaction.started", + "interaction.completed", + "agent.message.started", + "agent.message.completed", + "tool.created", + "tool.started", + "tool.completed", + "tool.failed", + "permission.requested", + "permission.decided", + "plan.updated", + "usage.updated", + "ide.document.changed", + "ide.file.opened", + "ide.file.saved", + "ide.file.closed", + "ide.run.executed", + "system.agent.crashed", + "system.proxy.error", + "agent.error", + "unknown_source_event" + ] + }, + "source": { + "title": "Source", + "type": "string" + }, + "study_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Study Id" + }, + "revision_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Revision Id" + }, + "enrollment_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enrollment Id" + }, + "research_session_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Research Session Id" + }, + "agent_run_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Run Id" + }, + "occurred_at": { + "format": "date-time", + "title": "Occurred At", + "type": "string" + }, + "monotonic_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Monotonic Ns" + }, + "emitter_id": { + "title": "Emitter Id", + "type": "string" + }, + "emitter_sequence": { + "title": "Emitter Sequence", + "type": "integer" + }, + "correlations": { + "$ref": "#/$defs/Correlations" + }, + "lifecycle_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lifecycle State" + }, + "payload": { + "additionalProperties": true, + "title": "Payload", + "type": "object" + }, + "metrics": { + "$ref": "#/$defs/EventMetrics" + }, + "privacy": { + "$ref": "#/$defs/PrivacySummary" + }, + "provenance": { + "$ref": "#/$defs/Provenance" + }, + "coverage": { + "$ref": "#/$defs/Coverage" + }, + "unknown_event_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Event Type" + }, + "unknown_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Source" + }, + "unknown_lifecycle_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Lifecycle State" + } + }, + "required": [ + "event_id", + "schema_version", + "event_type", + "source", + "occurred_at", + "emitter_id", + "emitter_sequence", + "provenance" + ], + "title": "CanonicalEventV1", + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://code4me.invalid/research/schema/canonical_event_v1.json" +} diff --git a/src/research/telemetry/validation.py b/src/research/telemetry/validation.py new file mode 100644 index 0000000..a263f7c --- /dev/null +++ b/src/research/telemetry/validation.py @@ -0,0 +1,239 @@ +"""Typed validation for canonical telemetry envelopes. + +Validation never raises for data: it returns a list of typed +:class:`TelemetryValidationError`. Unknown-but-preserved values produce a +``NEEDS_REVIEW`` finding rather than a hard rejection, while structural +problems (missing id, missing provenance, bad sequence) are errors. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any, Mapping, Union + +from pydantic import BaseModel, ConfigDict +from pydantic import ValidationError as PydanticValidationError + +from .enums import CanonicalEventType, EventSource +from .models import CanonicalEventV1 +from .privacy.classify import contains_secret_value + +__all__ = [ + "TelemetryValidationCode", + "TelemetryValidationError", + "validate_canonical_event", +] + +_SCHEMA_VERSION = "1" +_KNOWN_EVENT_TYPES = frozenset(event_type.value for event_type in CanonicalEventType) +_KNOWN_SOURCES = frozenset(source.value for source in EventSource) + + +class TelemetryValidationCode(str, Enum): + """Stable, machine-readable canonical-event validation reason codes.""" + + SCHEMA_INVALID = "SCHEMA_INVALID" + SCHEMA_VERSION_UNSUPPORTED = "SCHEMA_VERSION_UNSUPPORTED" + SCHEMA_VERSION_MISSING = "SCHEMA_VERSION_MISSING" + MISSING_EVENT_ID = "MISSING_EVENT_ID" + MISSING_PROVENANCE = "MISSING_PROVENANCE" + MISSING_EMITTER_ID = "MISSING_EMITTER_ID" + INVALID_EMITTER_SEQUENCE = "INVALID_EMITTER_SEQUENCE" + MISSING_OCCURRED_AT = "MISSING_OCCURRED_AT" + UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE" + UNKNOWN_SOURCE = "UNKNOWN_SOURCE" + SECRET_PRESENT = "SECRET_PRESENT" + PRIVACY_BLOCKED = "PRIVACY_BLOCKED" + + +class TelemetryValidationError(BaseModel): + """One typed validation finding.""" + + model_config = ConfigDict(extra="forbid") + + code: TelemetryValidationCode + field: str + message: str + needs_review: bool = False + + +def _issue( + code: TelemetryValidationCode, field: str, message: str, *, needs_review: bool = False +) -> TelemetryValidationError: + return TelemetryValidationError( + code=code, field=field, message=message, needs_review=needs_review + ) + + +def _validate_model(event: CanonicalEventV1) -> list[TelemetryValidationError]: + errors: list[TelemetryValidationError] = [] + + if event.schema_version != _SCHEMA_VERSION: + errors.append( + _issue( + TelemetryValidationCode.SCHEMA_VERSION_UNSUPPORTED, + "schema_version", + f"unsupported schema_version {event.schema_version!r}", + ) + ) + + if event.event_id is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_EVENT_ID, "event_id", "event_id is required" + ) + ) + + if event.provenance is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_PROVENANCE, + "provenance", + "provenance is required", + ) + ) + + if not event.emitter_id or not str(event.emitter_id).strip(): + errors.append( + _issue( + TelemetryValidationCode.MISSING_EMITTER_ID, + "emitter_id", + "emitter_id is required for per-emitter ordering", + ) + ) + + if event.emitter_sequence is None or event.emitter_sequence < 1: + errors.append( + _issue( + TelemetryValidationCode.INVALID_EMITTER_SEQUENCE, + "emitter_sequence", + "emitter_sequence must be a strictly positive integer", + ) + ) + + if event.occurred_at is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_OCCURRED_AT, + "occurred_at", + "occurred_at is required", + ) + ) + + if event.privacy.blocked: + errors.append( + _issue( + TelemetryValidationCode.PRIVACY_BLOCKED, + "privacy.blocked", + "the privacy filter blocked this event; a blocked event is never persistable", + ) + ) + + if event.event_type not in _KNOWN_EVENT_TYPES: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_EVENT_TYPE, + "event_type", + f"unrecognized event_type {event.event_type!r}", + needs_review=True, + ) + ) + elif event.unknown_event_type is not None: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_EVENT_TYPE, + "unknown_event_type", + f"source event type {event.unknown_event_type!r} was preserved", + needs_review=True, + ) + ) + + if event.source not in _KNOWN_SOURCES: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_SOURCE, + "source", + f"unrecognized source {event.source!r}", + needs_review=True, + ) + ) + elif event.unknown_source is not None: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_SOURCE, + "unknown_source", + f"source {event.unknown_source!r} was preserved", + needs_review=True, + ) + ) + + # Defense in depth: a filtered event must not still carry secret-shaped + # material anywhere it can be persisted. Provenance and correlation handles + # are attacker-influenced wire fields, so they are scanned too (not only the + # payload/metrics). + for field_name, value in ( + ("payload", event.payload), + ("metrics", event.metrics.model_dump(mode="json")), + ("provenance", event.provenance.model_dump(mode="json")), + ("correlations", event.correlations.model_dump(mode="json")), + ("lifecycle_state", event.lifecycle_state), + ("unknown_event_type", event.unknown_event_type), + ("unknown_source", event.unknown_source), + ("unknown_lifecycle_state", event.unknown_lifecycle_state), + ): + if contains_secret_value(value): + errors.append( + _issue( + TelemetryValidationCode.SECRET_PRESENT, + field_name, + f"{field_name} contains secret material and must be filtered first", + ) + ) + + return errors + + +def validate_canonical_event( + event_or_dict: Union[CanonicalEventV1, Mapping[str, Any]], +) -> list[TelemetryValidationError]: + """Return every typed validation finding for a canonical event.""" + if isinstance(event_or_dict, CanonicalEventV1): + return _validate_model(event_or_dict) + + if not isinstance(event_or_dict, Mapping): + return [ + _issue( + TelemetryValidationCode.SCHEMA_INVALID, + "", + "canonical event must be a CanonicalEventV1 or a mapping", + ) + ] + + try: + event = CanonicalEventV1.model_validate(dict(event_or_dict)) + except PydanticValidationError as error: + details = error.errors() + # A missing schema_version is its own typed reason: the envelope version + # is never defaulted, so "absent" must be distinguishable from + # "present but unsupported". + for detail in details: + location = tuple(detail.get("loc", ())) + if detail.get("type") == "missing" and location == ("schema_version",): + return [ + _issue( + TelemetryValidationCode.SCHEMA_VERSION_MISSING, + "schema_version", + "schema_version is required; it is never defaulted on ingestion", + ) + ] + first = details[0] if details else {} + field = ".".join(str(part) for part in first.get("loc", ())) + return [ + _issue( + TelemetryValidationCode.SCHEMA_INVALID, + field, + str(first.get("msg", "canonical event failed schema validation")), + ) + ] + + return _validate_model(event) diff --git a/src/response_models.py b/src/response_models.py index abb760c..113755c 100644 --- a/src/response_models.py +++ b/src/response_models.py @@ -36,6 +36,12 @@ class ResponseUser(Queries.CreateUser): default=None, description="Last authentication token used by the user" ) is_admin: bool = Field(False, description="Whether the user is an admin") + # Administrator-enabled researcher account (P3): the single researcher + # authority. Exposed so the website can gate researcher surfaces without a + # per-study role lookup. + can_research: bool = Field( + False, description="Whether an admin enabled this account for research" + ) @field_validator("preference", mode="before") @classmethod diff --git a/src/website/src/App.js b/src/website/src/App.js index 117e42c..45b7fc8 100644 --- a/src/website/src/App.js +++ b/src/website/src/App.js @@ -1,11 +1,24 @@ import React, { useState, useEffect } from "react"; import Auth from "./components/auth/Auth"; import Dashboard from "./pages/Dashboard"; +import ThemeToggle from "./components/common/ThemeToggle"; +import ResearchStudies from "./pages/research/ResearchStudies"; +import ResearchStudyEditor from "./pages/research/ResearchStudyEditor"; +import ResearchEnrollment from "./pages/research/ResearchEnrollment"; +import ResearchJoin from "./pages/research/ResearchJoin"; import "./App.css"; import { GoogleOAuthProvider } from "@react-oauth/google"; import { ThemeProvider } from "./context/ThemeContext"; import { getCurrentUser, logoutUser } from "./utils/api"; -import { BrowserRouter, Routes, Route, Navigate, useNavigate } from "react-router-dom"; +import { + BrowserRouter, + Routes, + Route, + Navigate, + NavLink, + Outlet, + useNavigate, +} from "react-router-dom"; import Start from "./pages/Start"; function App() { // we manage the state for the uer but setting it to null by default @@ -81,6 +94,128 @@ function App() { return children; }; + // Any signed-in account may reach the research authoring shell; the backend + // authorizes per study (a role grant for draft/publish) and reserves the + // deployment-wide surfaces (role grants, pilot/release operations, the kill + // switch) for administrators. The frontend only gates those genuinely + // admin-only surfaces; everything else defers to the server's 403. + const AdminRoute = ({ children }) => { + if (!user) return ; + if (!user.is_admin) return ; + return children; + }; + + const ResearchLayout = () => { + const navigate = useNavigate(); + const onLogoutWrapped = async () => { + await handleLogout(); + navigate("/", { replace: true }); + }; + const linkClass = ({ isActive }) => + `nav-item research-nav-link${isActive ? " active" : ""}`; + return ( +
+
+
+
+

Research Control Plane

+ + {user?.is_admin ? "Researcher" : "Restricted"} + +
+
+ + + {user ? user.name || user.email : "Loading user..."} + + + Dashboard + + +
+
+
+ +
+ + +
+ +
+
+
+ ); + }; + + // Participant-facing research shell. Unlike ResearchLayout this is not + // admin-gated: any signed-in account can redeem a study join code. + const ResearchParticipantLayout = () => { + const navigate = useNavigate(); + const onLogoutWrapped = async () => { + await handleLogout(); + navigate("/", { replace: true }); + }; + return ( +
+
+
+
+

Research Participant

+ Join a study +
+
+ + + {user ? user.name || user.email : "Loading user..."} + + + Dashboard + + +
+
+
+ +
+
+ +
+
+
+ ); + }; + const AuthPage = ({ mode }) => { const navigate = useNavigate(); const onAuth = (userData) => { @@ -138,6 +273,41 @@ function App() { } /> + + + + } + > + } /> + } /> + } + /> + {/* Enrollment manages the deployment-wide join code and the + study index; both backing endpoints are require_admin. */} + + + + } + /> + + + + + } + > + } /> + } />
diff --git a/src/website/src/__tests__/App.routing.test.js b/src/website/src/__tests__/App.routing.test.js new file mode 100644 index 0000000..c90426e --- /dev/null +++ b/src/website/src/__tests__/App.routing.test.js @@ -0,0 +1,63 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import App from "../App"; +import * as api from "../utils/api"; + +jest.mock("../utils/api"); +// Isolate the route guards: the pages themselves have their own suites. +jest.mock("../pages/Dashboard", () => () =>
DASHBOARD-PAGE
); +jest.mock("../components/auth/Auth", () => () =>
LOGIN-PAGE
); +jest.mock("../pages/research/ResearchStudies", () => () =>
STUDIES-PAGE
); +jest.mock("../pages/research/ResearchStudyEditor", () => () => ( +
EDITOR-PAGE
+)); +jest.mock("../pages/research/ResearchEnrollment", () => () => ( +
ENROLLMENT-PAGE
+)); +jest.mock("../pages/research/ResearchJoin", () => () =>
JOIN-PAGE
); + +const STUDY_ID = "11111111-1111-1111-1111-111111111111"; + +const renderAt = (path) => { + window.history.pushState({}, "", path); + return render(); +}; + +beforeEach(() => { + jest.clearAllMocks(); + process.env.REACT_APP_GOOGLE_CLIENT_ID = "test-client-id"; + api.getCurrentUser.mockResolvedValue({ + ok: true, + user: { email: "researcher@example.com", is_admin: false }, + config: {}, + }); + api.logoutUser.mockResolvedValue({ ok: true }); +}); + +test("a non-admin authenticated user can reach the study editor route", async () => { + renderAt(`/research/studies/${STUDY_ID}/editor`); + + expect(await screen.findByText("EDITOR-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("DASHBOARD-PAGE")).not.toBeInTheDocument(); +}); + +test("a non-admin authenticated user can reach the studies route", async () => { + renderAt("/research/studies"); + + expect(await screen.findByText("STUDIES-PAGE")).toBeInTheDocument(); +}); + +test("the admin-only enrollment surface stays admin-gated", async () => { + renderAt("/research/enrollment"); + + expect(await screen.findByText("DASHBOARD-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("ENROLLMENT-PAGE")).not.toBeInTheDocument(); +}); + +test("an unauthenticated visitor is redirected to login", async () => { + api.getCurrentUser.mockResolvedValue({ ok: false }); + renderAt(`/research/studies/${STUDY_ID}/editor`); + + expect(await screen.findByText("LOGIN-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("EDITOR-PAGE")).not.toBeInTheDocument(); +}); diff --git a/src/website/src/components/analytics/AgentAnalytics.css b/src/website/src/components/analytics/AgentAnalytics.css index c092deb..1e33671 100644 --- a/src/website/src/components/analytics/AgentAnalytics.css +++ b/src/website/src/components/analytics/AgentAnalytics.css @@ -63,9 +63,11 @@ .agent-graph-rail::before { position: absolute; top: 0; bottom: -6px; left: 5px; width: 2px; background: var(--border-color); content: ""; } .agent-graph-rail::after { position: absolute; top: 18px; left: 1px; width: 10px; height: 10px; border: 2px solid var(--card-background); border-radius: 50%; background: #0f766e; box-shadow: 0 0 0 1px #0f766e; content: ""; } .agent-graph-rail.tool::after { background: #d97706; box-shadow: 0 0 0 1px #d97706; } +.agent-graph-rail.observed::after { background: #6d28d9; box-shadow: 0 0 0 1px #6d28d9; } .agent-graph-rail.failure::after { background: #c2410c; box-shadow: 0 0 0 1px #c2410c; } .agent-graph-node { display: flex; justify-content: space-between; gap: 14px; width: 100%; padding: 8px 10px; border: 1px solid rgba(15, 118, 110, .25); border-radius: 6px; background: rgba(15, 118, 110, .06); } .agent-graph-node.tool { border-color: rgba(217, 119, 6, .3); background: rgba(217, 119, 6, .06); } +.agent-graph-node.observed { border-color: rgba(109, 40, 217, .3); background: rgba(109, 40, 217, .06); } .agent-graph-node.failure { border-color: rgba(194, 65, 12, .4); background: rgba(194, 65, 12, .07); } .agent-graph-node-main { min-width: 0; } .agent-graph-node-main strong, .agent-graph-node-main small { display: block; } diff --git a/src/website/src/components/analytics/AgentAnalytics.js b/src/website/src/components/analytics/AgentAnalytics.js index 810d230..0fffaac 100644 --- a/src/website/src/components/analytics/AgentAnalytics.js +++ b/src/website/src/components/analytics/AgentAnalytics.js @@ -6,6 +6,8 @@ import "./AgentAnalytics.css"; const number = (value) => Number(value || 0).toLocaleString(); const percent = (value) => `${(Number(value || 0) * 100).toFixed(1)}%`; const milliseconds = (value) => `${Math.round(Number(value || 0))}ms`; +// A missing measurement is rendered as unavailable, never as a zero. +const ifAvailable = (value, render) => (value == null ? "unavailable" : render(value)); const mergeOptions = (previous, values) => [...new Set([...previous, ...values])].sort(); @@ -51,14 +53,16 @@ const EventGraph = ({ events }) => { const depth = eventDepth(event, spanIndexes, events); const isTool = event.event_type.includes("tool"); const isFailure = event.event_type.includes("failed") || event.upstream_status >= 400; + const isObserved = event.source === "acp-proxy"; + const kind = isObserved ? "observed" : isTool ? "tool" : "model"; return (
-
-
+
+
{event.event_index} {event.event_type} - {event.source || "unknown source"}{event.tool_name ? ` · ${event.tool_name}` : event.model ? ` · ${event.model}` : ""} + {event.source || "unknown source"}{event.detail ? ` · ${event.detail}` : ""}{event.tool_name ? ` · ${event.tool_name}` : event.model ? ` · ${event.model}` : ""}
{event.latency_ms ? {milliseconds(event.latency_ms)} : null} @@ -173,17 +177,17 @@ const AgentAnalytics = ({ timeWindow = "7d" }) => { - +
-
Provider input tokens{number(summary.provider_input_tokens)}reported by model provider
-
Conversation context~{number(summary.conversation_tokens_estimated)}estimated from message bytes
-
Tool schema~{number(summary.tool_schema_tokens_estimated)}estimated from schema bytes
-
Tool results~{number(summary.tool_result_tokens_estimated)}estimated from result bytes
-
Model output{number(summary.model_output_tokens)}reported completion tokens
+
Provider input tokens{ifAvailable(summary.provider_input_tokens, number)}reported by model provider
+
Conversation context{ifAvailable(summary.conversation_tokens_estimated, (v) => `~${number(v)}`)}estimated from message bytes
+
Tool schema{ifAvailable(summary.tool_schema_tokens_estimated, (v) => `~${number(v)}`)}estimated from schema bytes
+
Tool results{ifAvailable(summary.tool_result_tokens_estimated, (v) => `~${number(v)}`)}estimated from result bytes
+
Model output{ifAvailable(summary.model_output_tokens, number)}reported completion tokens
Successful calls{number(summary.successful_model_calls)}{number(summary.rate_limit_retries)} rate-limit retries
diff --git a/src/website/src/components/analytics/AgentResults.js b/src/website/src/components/analytics/AgentResults.js index a8548d6..706ac49 100644 --- a/src/website/src/components/analytics/AgentResults.js +++ b/src/website/src/components/analytics/AgentResults.js @@ -15,12 +15,6 @@ const BAR_METRICS = [ better: "higher", format: (v) => `${((v || 0) * 100).toFixed(1)}%`, }, - { - key: "edit_acceptance_rate", - label: "Edit acceptance rate", - better: "higher", - format: (v) => `${((v || 0) * 100).toFixed(1)}%`, - }, { key: "avg_steps", label: "Avg steps / task", @@ -223,9 +217,7 @@ const ArmCard = ({ arm }) => {
Edits accepted: - - {m.accepted_edits || 0} of {m.total_edits || 0} - + unavailable
@@ -235,10 +227,6 @@ const ArmCard = ({ arm }) => { label="Completion vs baseline" value={arm.uplift.completion_rate_change_pct} /> - { @@ -35,7 +36,7 @@ const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { } ]; - // Add admin-only navigation items + // Administrator-only navigation items. if (user?.is_admin) { navigationItems.push({ id: 'studies', @@ -51,19 +52,16 @@ const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { description: 'Create and manage server/config modules', adminOnly: true }); + } + + // Researcher surfaces: administrators and administrator-enabled researchers. + if (user?.is_admin || user?.can_research) { navigationItems.push({ id: 'agent-profiles', label: 'Agent Profiles', icon: '🛠️', - description: 'Agent runtime, provider, tools and policy variants', - adminOnly: true - }); - navigationItems.push({ - id: 'agent-assignments', - label: 'Agent Assignments', - icon: '🎲', - description: 'A/B bucket distribution and manual pins', - adminOnly: true + description: 'Agent runtime, connection, tools and policy variants', + researcherOnly: true }); } @@ -71,9 +69,11 @@ const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { ); diff --git a/src/website/src/components/analytics/StudyManagement.js b/src/website/src/components/analytics/StudyManagement.js index 6b86aa8..d8d1292 100644 --- a/src/website/src/components/analytics/StudyManagement.js +++ b/src/website/src/components/analytics/StudyManagement.js @@ -340,9 +340,11 @@ const StudyManagement = ({ user }) => { {config.metrics.total_queries}
- Acceptance: + Completion acceptance: - {(config.metrics.acceptance_rate * 100).toFixed(1)}% + {config.metrics.acceptance_rate == null + ? "unavailable" + : `${(config.metrics.acceptance_rate * 100).toFixed(1)}%`}
@@ -357,9 +359,10 @@ const StudyManagement = ({ user }) => {
vs Baseline: - - {config.vs_baseline.acceptance_rate_uplift_pct > 0 ? '+' : ''} - {config.vs_baseline.acceptance_rate_uplift_pct.toFixed(1)}% + + {config.vs_baseline.acceptance_rate_uplift_pct == null + ? "unavailable" + : `${config.vs_baseline.acceptance_rate_uplift_pct > 0 ? '+' : ''}${config.vs_baseline.acceptance_rate_uplift_pct.toFixed(1)}%`}
diff --git a/src/website/src/pages/AgentProfiles.css b/src/website/src/pages/AgentProfiles.css index b666bd1..a708145 100644 --- a/src/website/src/pages/AgentProfiles.css +++ b/src/website/src/pages/AgentProfiles.css @@ -109,6 +109,67 @@ flex-wrap: wrap; } +/* Derived, read-only distribution verification detail. */ +.profile-derived { + display: flex; + flex-direction: column; + gap: 8px; + border: 1px solid var(--border-color); + border-radius: 6px; + background-color: var(--background-color); + padding: 12px; +} + +.profile-badge { + align-self: flex-start; + border-radius: 999px; + font-size: 11px; + font-weight: 700; + padding: 3px 10px; + text-transform: uppercase; +} + +.profile-badge.verified { + color: #0b6b35; + background: rgba(11, 107, 53, 0.1); + border: 1px solid rgba(11, 107, 53, 0.25); +} + +.profile-badge.unverified { + color: #92500a; + background: rgba(146, 80, 10, 0.1); + border: 1px solid rgba(146, 80, 10, 0.3); +} + +.profile-derived-fields { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); + gap: 10px; + margin: 0; +} + +.profile-derived-fields dt { + color: var(--text-secondary); + font-size: 11px; + font-weight: 700; + text-transform: uppercase; +} + +.profile-derived-fields dd { + color: var(--text-color); + font-size: 13px; + margin: 3px 0 0; + overflow-wrap: anywhere; +} + +.profile-derived-release { + color: var(--text-secondary); + display: block; + font-size: 12px; + margin-top: 2px; + overflow-wrap: anywhere; +} + .primary-button, .secondary-button, .danger-button { diff --git a/src/website/src/pages/AgentProfiles.js b/src/website/src/pages/AgentProfiles.js index 9a4fcaf..88e254e 100644 --- a/src/website/src/pages/AgentProfiles.js +++ b/src/website/src/pages/AgentProfiles.js @@ -3,7 +3,9 @@ import { createAgentProfile, deleteAgentProfile, getAgentAvailableTools, + getAgentDistributions, getAgentProfiles, + getProviderConnections, updateAgentProfile, } from "../utils/api"; import "./AgentProfiles.css"; @@ -34,12 +36,15 @@ const APPROVAL_POLICIES = [ { value: "auto", label: "Auto-approve" }, ]; +// Only the fields the backend accepts. Legacy BYOA/provider fields +// (base_url/api_key_ref/distribution_mode/agent_package/...) were removed from +// the contract and are rejected via extra="forbid". const EMPTY_FORM = { name: "", model: "", framework_version: "code4me2-agent", - base_url: "", - api_key_ref: "", + connection_id: "", + release_id: "", tools: [], approval_policy: "per_step", max_steps: 15, @@ -60,16 +65,31 @@ const formatTools = (toolsJson) => { } }; +const formatPlatforms = (platforms) => { + if (!Array.isArray(platforms) || platforms.length === 0) return "none declared"; + return platforms + .map((platform) => `${platform.os || "?"}/${platform.arch || "?"}`) + .join(", "); +}; + const getProfileId = (profile) => profile.profile_id || profile.id || profile.name; -const AgentProfiles = () => { +// A profile may select one of the connection's allowed models. The backend +// re-validates this, so the UI constraint is only to avoid an obvious 422. +const connectionModels = (connection) => + Array.isArray(connection?.models) ? connection.models : []; + +const AgentProfiles = ({ user = {} }) => { const [profiles, setProfiles] = useState([]); const [availableTools, setAvailableTools] = useState([]); + const [connections, setConnections] = useState([]); + const [distributions, setDistributions] = useState([]); const [form, setForm] = useState(EMPTY_FORM); const [editingProfileId, setEditingProfileId] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSaving, setIsSaving] = useState(false); const [error, setError] = useState(""); + const [fieldErrors, setFieldErrors] = useState([]); const [notice, setNotice] = useState(""); const [isToolsOpen, setIsToolsOpen] = useState(false); const toolsDropdownRef = useRef(null); @@ -82,11 +102,13 @@ const AgentProfiles = () => { const loadProfiles = async () => { setIsLoading(true); setError(""); + setFieldErrors([]); const response = await getAgentProfiles(); if (response.ok) { setProfiles(Array.isArray(response.data) ? response.data : []); } else { setError(response.error); + setFieldErrors(Array.isArray(response.errors) ? response.errors : []); } setIsLoading(false); }; @@ -95,6 +117,61 @@ const AgentProfiles = () => { loadProfiles(); }, []); + // Provider connections the caller may use (admin: all; researcher: granted). + useEffect(() => { + let cancelled = false; + getProviderConnections().then((response) => { + if (!cancelled && response && response.ok) { + setConnections(Array.isArray(response.data) ? response.data : []); + } + }); + return () => { + cancelled = true; + }; + }, []); + + // The shared, non-admin-readable release catalogue is derived from the + // server's distribution views: each carries the resolved release identity and + // the derived `verified` flag, so no admin-only release endpoint is needed. + useEffect(() => { + let cancelled = false; + getAgentDistributions().then((response) => { + if (!cancelled && response && response.ok) { + setDistributions(Array.isArray(response.data) ? response.data : []); + } + }); + return () => { + cancelled = true; + }; + }, []); + + const releaseCatalogue = useMemo(() => { + const byId = new Map(); + distributions.forEach((distribution) => { + const releaseId = distribution.release_id; + if (!releaseId) return; + const verified = Boolean(distribution.verified); + const existing = byId.get(releaseId); + if (!existing || (verified && !existing.verified)) { + byId.set(releaseId, { + release_id: releaseId, + release_version: distribution.release_version, + verified, + // Approval options this release's conformance evidence verifies. + // Missing on older servers: treat as unconstrained. + verified_approval_options: Array.isArray( + distribution.verified_approval_options, + ) + ? distribution.verified_approval_options + : null, + }); + } + }); + return Array.from(byId.values()).sort((a, b) => + String(a.release_id).localeCompare(String(b.release_id)), + ); + }, [distributions]); + // The selectable tool set depends on the runtime, so reload it whenever the // runtime changes rather than showing tools the agent can't actually call. useEffect(() => { @@ -121,36 +198,64 @@ const AgentProfiles = () => { return () => document.removeEventListener("mousedown", handleClickOutside); }, [isToolsOpen]); + const selectedConnection = connections.find( + (connection) => connection.connection_id === form.connection_id, + ); + const selectableModels = connectionModels(selectedConnection); + const selectedRelease = releaseCatalogue.find( + (release) => release.release_id === form.release_id, + ); + // The approval options the selected release's evidence verifies; null means + // an older catalogue with no evidence detail (leave unconstrained). + const verifiedApprovals = + selectedRelease && Array.isArray(selectedRelease.verified_approval_options) + ? new Set(selectedRelease.verified_approval_options) + : null; + const validateForm = () => { if (!form.name.trim()) return "Profile name is required."; if (!/^[a-z0-9_-]+$/.test(form.name.trim())) { return "Use lowercase letters, numbers, hyphens, or underscores for the name."; } - if (!form.model.trim()) return "Model is required."; + if (!form.connection_id) { + return "Select a provider connection. An administrator grants connections."; + } + if (!form.model.trim()) return "Select a model."; + if ( + selectedConnection && + selectableModels.length > 0 && + !selectableModels.includes(form.model.trim()) + ) { + return "The selected model is not allowed by this provider connection."; + } + if (!form.release_id) { + return "Select a registered release to pin this profile to an artifact."; + } if (form.temperature !== "" && form.temperature !== null) { const t = Number(form.temperature); if (Number.isNaN(t) || t < TEMPERATURE_MIN || t > TEMPERATURE_MAX) { return "Temperature must be a number between 0 and 2 (or blank)."; } } - if (form.base_url.trim() && !/^https?:\/\//i.test(form.base_url.trim())) { - return "Base URL must start with http:// or https://"; + if (!user?.is_admin && selectedRelease && !selectedRelease.verified) { + return "This release is not verified. Only an administrator can pin an unverified release."; } - // Guard against the obvious mistake of pasting the key itself. The backend - // rejects this too; catching it here avoids a round-trip and, more - // importantly, avoids sending a real credential over the wire at all. - const keyRef = form.api_key_ref.trim(); - if (keyRef && (keyRef.length > 128 || /[-.\s/:]/.test(keyRef))) { - return "API key ref must be the NAME of an environment variable (e.g. OPENAI_API_KEY), not the key itself."; + if (verifiedApprovals && !verifiedApprovals.has(form.approval_policy)) { + return "The selected approval policy is not verified for this release."; } return ""; }; + const clearMessages = () => { + setError(""); + setFieldErrors([]); + setNotice(""); + }; + const resetForm = () => { setForm(EMPTY_FORM); setEditingProfileId(null); - setError(""); - setNotice(""); + clearMessages(); }; const handleChange = (event) => { @@ -161,6 +266,23 @@ const AgentProfiles = () => { })); }; + const handleConnectionChange = (event) => { + const connectionId = event.target.value; + setForm((current) => { + const connection = connections.find( + (candidate) => candidate.connection_id === connectionId, + ); + const models = connectionModels(connection); + // A stale model from the previous connection would fail validation. + const keepModel = models.length === 0 || models.includes(current.model); + return { + ...current, + connection_id: connectionId, + model: keepModel ? current.model : "", + }; + }); + }; + const handleTemperatureChange = (event) => { const { value } = event.target; if (value === "") { @@ -200,21 +322,19 @@ const AgentProfiles = () => { const validationError = validateForm(); if (validationError) { setError(validationError); + setFieldErrors([]); return; } setIsSaving(true); - setError(""); - setNotice(""); + clearMessages(); const payload = { name: form.name.trim(), model: form.model.trim(), framework_version: form.framework_version, - // Blank means "use the server default upstream", which the backend - // represents as NULL rather than an empty string. - base_url: form.base_url.trim() || null, - api_key_ref: form.api_key_ref.trim() || null, + connection_id: form.connection_id, + release_id: form.release_id.trim() || null, // The API takes a JSON string, not an array. tools_json: JSON.stringify(form.tools), approval_policy: form.approval_policy, @@ -238,13 +358,15 @@ const AgentProfiles = () => { await loadProfiles(); } else { setError(response.error); + setFieldErrors(Array.isArray(response.errors) ? response.errors : []); } setIsSaving(false); }; - const handleEdit = (profile) => { - setEditingProfileId(getProfileId(profile)); + // Map a stored profile onto the editable form. The response exposes a + // `connection` summary plus the resolved release pin and derived flags. + const formFromProfile = (profile) => { let selectedTools = []; try { const parsed = JSON.parse(profile.tools_json || "[]"); @@ -252,12 +374,13 @@ const AgentProfiles = () => { } catch (_) { selectedTools = []; } - setForm({ + return { name: profile.name || "", model: profile.model || "", framework_version: profile.framework_version || "code4me2-agent", - base_url: profile.base_url || "", - api_key_ref: profile.api_key_ref || "", + connection_id: + profile.connection?.connection_id || profile.connection_id || "", + release_id: profile.release_id || "", tools: selectedTools, approval_policy: profile.approval_policy || "per_step", max_steps: profile.max_steps || 15, @@ -266,14 +389,31 @@ const AgentProfiles = () => { profile.max_context_tokens === undefined ? "" : profile.max_context_tokens, - is_active: profile.is_active !== undefined ? Boolean(profile.is_active) : true, + is_active: + profile.is_active !== undefined ? Boolean(profile.is_active) : true, temperature: profile.temperature === null || profile.temperature === undefined ? "" : profile.temperature, - }); - setError(""); - setNotice(""); + }; + }; + + const handleEdit = (profile) => { + setEditingProfileId(getProfileId(profile)); + setForm(formFromProfile(profile)); + clearMessages(); + }; + + // Clone populates the *create* form with an existing profile's configuration + // under a new name, so a researcher can express v1 vs v2 as two templates that + // differ only in their release. + const handleClone = (profile) => { + setEditingProfileId(null); + setForm({ ...formFromProfile(profile), name: `${profile.name || "profile"}-copy` }); + clearMessages(); + setNotice( + `Cloning "${profile.name}". Change the release pin and save to create a new profile.`, + ); }; const handleDelete = async (profile) => { @@ -286,8 +426,7 @@ const AgentProfiles = () => { if (!confirmed) return; setIsSaving(true); - setError(""); - setNotice(""); + clearMessages(); const response = await deleteAgentProfile(profileId); if (response.ok) { @@ -296,6 +435,7 @@ const AgentProfiles = () => { await loadProfiles(); } else { setError(response.error); + setFieldErrors(Array.isArray(response.errors) ? response.errors : []); } setIsSaving(false); @@ -306,6 +446,15 @@ const AgentProfiles = () => { (f) => f.value === form.framework_version, ); const toolsIgnored = form.framework_version === "codex"; + const editingProfile = editingProfileId + ? profiles.find((profile) => getProfileId(profile) === editingProfileId) || + null + : null; + + const releaseLabel = (release) => + `${release.release_id}${release.release_version ? ` · v${release.release_version}` : ""}${ + release.verified ? "" : " (unverified)" + }`; return (
@@ -314,9 +463,10 @@ const AgentProfiles = () => {

Agent Profiles

Define the experiment arms for agent runs: which runtime, which - OpenAI-compatible provider, which model, tools, approval policy and - step limit. A profile's settings are snapshotted onto each task when - it starts, so editing a profile never changes tasks already running. + authorized provider connection, which model, tools, approval policy + and step limit. A profile's settings are snapshotted onto each task + when it starts, so editing a profile never changes tasks already + running.

+ + +
+

+ The join code is study-scoped and shared by every participant. Loading + it does not enroll your account; use “Enroll this account” to mint + (or reuse) your own enrollment for inspection. +

+ + + {isLoading ? ( +

Loading revision…

+ ) : ( + revision && ( +
+

Revision #{revision.revision_number}

+
+
+
Status
+
{revision.status}
+
+
+
Digest
+
+ {shortId(revision.protocol_digest)} +
+
+
+
Published
+
{formatDateTime(revision.published_at)}
+
+
+
+ ) + )} + +
+

Join code

+

+ {studyJoinCode + ? "Study join code. Share it with participants after they install the plugin and sign in." + : studyJoinCodeMissing + ? "This server does not expose the study join-code endpoint yet, so the per-account enrollment id is shown as a fallback." + : "Load the study join code to get the value participants enter after signing up and signing in."} +

+ {displayJoinCode ? ( +
+ + {displayJoinCode} + + +
+ ) : ( +

No join code loaded yet.

+ )} + {(studyJoinCode || studyJoinCodeStatus || studyJoinCodeRevisionId) && ( +
+
+
Scope
+
{studyJoinCode ? "Study" : "Account fallback"}
+
+
+
Status
+
{studyJoinCodeStatus || "—"}
+
+
+
Revision
+
+ {studyJoinCodeRevisionId + ? shortId(studyJoinCodeRevisionId) + : "—"} +
+
+
+ )} +
+ + {enrollment && ( +
+

Account enrollment

+

+ Your signed-in account’s enrollment in this revision. This is not the + shareable study code. +

+
+
+
Status
+
{enrollment.status}
+
+
+
Participant code
+
{enrollment.participant_code || "—"}
+
+
+
Eligible
+
+ {enrollment.eligible === null || enrollment.eligible === undefined + ? "unknown" + : String(enrollment.eligible)} +
+
+
+
+ +
+
+ )} + +
+

Participant instructions

+
    +
  1. + Install the Code4me2 plugin from the{" "} + + JetBrains Marketplace + {" "} + (or from the study ZIP when a packaged build is provided). +
    2. +
  2. Sign up or sign in with your Code4me2 account.
    3. +
  3. Enter the join code in the research/enrollment panel.
    4. +
  4. Accept the study policy to activate enrollment.
    5. +
+
+ +
+

Agent packaging

+ {conditions.length === 0 ? ( +

+ Load a revision to see whether its conditions use packaged agents or + bring-your-own-agent (BYOA) runtimes. +

+ ) : ( + <> +
+
+
Packaged conditions
+
{packagedCount}
+
+
+
BYOA conditions
+
{byoaCount}
+
+
+
+ + + + + + + + + + + + {conditions.map((condition) => { + const resolved = condition.resolved_distribution || {}; + const distribution = + distributionById[condition.distribution_id] || {}; + const overrides = condition.declared_overrides || {}; + const packaged = isPackagedCondition( + condition, + distributionById, + ); + const agentLabel = + overrides.agent_profile || + resolved.agent_id || + distribution.name || + "—"; + const releaseLabel = packaged + ? resolved.release_id || + distribution.release_id || + resolved.version || + distribution.release_version || + "unpinned" + : resolved.agent_command || + resolved.agent_package || + distribution.agent_command || + distribution.agent_package || + "participant-installed"; + return ( + + + + + + + ); + })} + +
+ Agent distribution per condition +
ConditionAgentDistributionRelease
{condition.name || condition.condition_id}{agentLabel} + + {packaged ? "Packaged" : "BYOA"} + + {releaseLabel}
+
+ + )} + {packagesUnavailable ? ( +

+ Runtime package registry unavailable; distribution is inferred from + the protocol and the distribution registry. +

+ ) : ( +

+ {packages.length} runtime package + {packages.length === 1 ? "" : "s"} registered on the server. +

+ )} +
+ + ); +}; + +export default ResearchEnrollment; diff --git a/src/website/src/pages/research/ResearchJoin.js b/src/website/src/pages/research/ResearchJoin.js new file mode 100644 index 0000000..8f4bffb --- /dev/null +++ b/src/website/src/pages/research/ResearchJoin.js @@ -0,0 +1,301 @@ +import React, { useState } from "react"; +import { Link } from "react-router-dom"; +import { + getMyResearchEnrollments, + redeemResearchJoinCode, + resolveResearchJoinCode, +} from "../../utils/api"; +import { shortId } from "./protocol"; +import "./research.css"; +import "./ResearchEnrollment.css"; + +const ResearchJoin = () => { + const [code, setCode] = useState(""); + const [study, setStudy] = useState(null); + const [acceptConsent, setAcceptConsent] = useState(false); + const [enrollment, setEnrollment] = useState(null); + const [otherStudy, setOtherStudy] = useState(null); + const [isBusy, setIsBusy] = useState(false); + const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); + + const trimmedCode = () => code.trim(); + + const resetResults = () => { + setStudy(null); + setEnrollment(null); + setOtherStudy(null); + setAcceptConsent(false); + setError(""); + setNotice(""); + }; + + // The `/participants/me` projection names the revision `study_revision_id`; + // normalize it so the same entry can be rendered by the enrollment card. + const toEnrollmentView = (record) => ({ + enrollment_id: record.enrollment_id || "", + study_id: record.study_id || "", + revision_id: record.study_revision_id || record.revision_id || "", + status: record.status || "", + }); + + const handleResolve = async (event) => { + event.preventDefault(); + const value = trimmedCode(); + if (!value) { + setError("Enter the join code from your study invitation."); + return; + } + setIsBusy(true); + resetResults(); + const response = await resolveResearchJoinCode(value); + if (!(response && response.ok)) { + setError( + (response && response.error) || "That join code could not be resolved.", + ); + setIsBusy(false); + return; + } + const resolved = response.data || {}; + + // Resolve the code first, then branch on the caller's own enrollments so an + // already-active participant is never asked to consent a second time. The + // server only returns the caller's own study-local projections. + const enrollmentsResponse = await getMyResearchEnrollments(); + const enrollments = + enrollmentsResponse && + enrollmentsResponse.ok && + Array.isArray(enrollmentsResponse.data) + ? enrollmentsResponse.data + : []; + + const matchesCode = (record) => { + const revisionId = resolved.revision && resolved.revision.revisionId; + const studyId = resolved.study && resolved.study.studyId; + if (revisionId) { + return record.study_revision_id === revisionId; + } + if (studyId) { + return record.study_id === studyId; + } + return false; + }; + + const activeForCode = enrollments.find( + (record) => record.status === "ACTIVE" && matchesCode(record), + ); + if (activeForCode) { + setEnrollment(toEnrollmentView(activeForCode)); + setNotice( + "You are already enrolled in this study. Your enrollment is active — no further action is needed.", + ); + setIsBusy(false); + return; + } + + const otherActive = enrollments.find( + (record) => + record.status === "ACTIVE" && + (!(resolved.study && resolved.study.studyId) || + record.study_id !== resolved.study.studyId), + ); + if (otherActive) { + setOtherStudy(toEnrollmentView(otherActive)); + setError( + "You are already enrolled in another research study. Leave that study first, or view your current enrollment below.", + ); + setIsBusy(false); + return; + } + + setStudy(resolved); + setNotice("Code found. Review the study policy and accept it to join."); + setIsBusy(false); + }; + + const handleRedeem = async () => { + const value = trimmedCode(); + if (!value) { + setError("Enter the join code from your study invitation."); + return; + } + if (!acceptConsent) { + setError("Accept the study policy to join."); + return; + } + setIsBusy(true); + setError(""); + setNotice(""); + const response = await redeemResearchJoinCode(value, true); + if (response && response.ok) { + setEnrollment(response.data); + setOtherStudy(null); + setStudy(null); + setNotice("Enrollment complete."); + } else { + setEnrollment(null); + setError( + (response && response.error) || "That join code could not be redeemed.", + ); + } + setIsBusy(false); + }; + + return ( +
+
+
+

Join a study

+

+ Enter the code from your study invitation. You can review the study + and its policy before redeeming the code. +

+
+ + Back to dashboard + +
+ + {(error || notice) && ( +
+ {error || notice} +
+ )} + +
+

Enter join code

+ +

+ The join code is case-insensitive and is provided by the researcher. +

+
+ +
+
+ + {study && ( +
+

{study.name || "Research study"}

+

+ Confirm this is the study you were invited to before enrolling. +

+
+
+
Study
+
{shortId(study.study && study.study.studyId)}
+
+
+
Revision
+
{shortId(study.revision && study.revision.revisionId)}
+
+
+
Status
+
{(study.revision && study.revision.status) || "—"}
+
+
+ {study.policyText ? ( +
+

Study policy

+

{study.policyText}

+
+ ) : null} + +
+ +
+
+ )} + + {enrollment && ( +
+

Enrollment status

+
+
+
Status
+
{enrollment.status || "—"}
+
+
+
Study
+
{shortId(enrollment.study_id)}
+
+
+
Revision
+
+ {shortId(enrollment.revision_id)} +
+
+
+
Enrollment
+
+ {shortId(enrollment.enrollment_id)} +
+
+
+
+ )} + + {otherStudy && ( +
+

Your current enrollment

+

+ You are already participating in another study. Leave that study + before joining a new one. +

+
+
+
Status
+
{otherStudy.status || "—"}
+
+
+
Study
+
{shortId(otherStudy.study_id)}
+
+
+
Revision
+
+ {shortId(otherStudy.revision_id)} +
+
+
+
Enrollment
+
+ {shortId(otherStudy.enrollment_id)} +
+
+
+
+ )} +
+ ); +}; + +export default ResearchJoin; diff --git a/src/website/src/pages/research/ResearchStudies.css b/src/website/src/pages/research/ResearchStudies.css new file mode 100644 index 0000000..4ca584f --- /dev/null +++ b/src/website/src/pages/research/ResearchStudies.css @@ -0,0 +1,57 @@ +.research-study-scope { + max-width: 720px; +} + +.research-study-name { + color: var(--text-color); + font-weight: 600; +} + +.research-study-id { + color: var(--text-secondary); + font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace; + font-size: 12px; +} + +.research-code-inline { + background-color: var(--background-color); + border: 1px dashed var(--primary-color); + border-radius: 4px; + color: var(--text-color); + font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace; + font-size: 13px; + font-weight: 700; + padding: 3px 8px; + overflow-wrap: anywhere; +} + +.research-row-selected { + background-color: var(--hover-color); +} + +.research-draft-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 10px; +} + +.research-draft-item { + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 6px; + padding: 12px 14px; + flex-wrap: wrap; +} + +.research-draft-meta { + color: var(--text-secondary); + font-size: 12px; + margin-left: 10px; +} diff --git a/src/website/src/pages/research/ResearchStudies.js b/src/website/src/pages/research/ResearchStudies.js new file mode 100644 index 0000000..9bf915f --- /dev/null +++ b/src/website/src/pages/research/ResearchStudies.js @@ -0,0 +1,774 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { useNavigate, useSearchParams } from "react-router-dom"; +import { + createResearchStudy, + getResearchDraft, + getResearchEnrollmentCoverage, + getResearchExposures, + listResearchDrafts, + listResearchRevisions, + listResearchStudies, + publishResearchDraft, + retireResearchRevision, + validateResearchProtocol, +} from "../../utils/api"; +import { formatDateTime, shortId } from "./protocol"; +import "./research.css"; +import "./ResearchStudies.css"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +const RevisionStatusBadge = ({ status }) => { + const normalized = String(status || "").toLowerCase(); + return ( + + {status || "UNKNOWN"} + + ); +}; + +const ResearchStudies = () => { + const navigate = useNavigate(); + const [searchParams] = useSearchParams(); + + const [studyId, setStudyId] = useState(searchParams.get("study_id") || ""); + const [appliedStudyId, setAppliedStudyId] = useState( + searchParams.get("study_id") || "", + ); + const [studies, setStudies] = useState([]); + const [studiesMissing, setStudiesMissing] = useState(false); + const [studiesForbidden, setStudiesForbidden] = useState(false); + + const [revisions, setRevisions] = useState([]); + const [drafts, setDrafts] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [isBusy, setIsBusy] = useState(false); + const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); + const [validationErrors, setValidationErrors] = useState([]); + + const [selectedRevisionId, setSelectedRevisionId] = useState(""); + const [exposures, setExposures] = useState([]); + const [enrollmentCoverage, setEnrollmentCoverage] = useState(null); + const [coverageError, setCoverageError] = useState(""); + + // "New study" mints the study UUID server-side so the researcher never pastes + // one. The form stays inline (no modal) and degrades to the manual UUID entry + // below when the create endpoint is unavailable on an older server. + const [isCreatingStudy, setIsCreatingStudy] = useState(false); + const [newStudyName, setNewStudyName] = useState(""); + const [newStudyDescription, setNewStudyDescription] = useState(""); + + const selectedRevision = useMemo( + () => + revisions.find((revision) => revision.revision_id === selectedRevisionId) || + null, + [revisions, selectedRevisionId], + ); + + // Study index may not exist on the server; the guarded call degrades to + // manual study-id entry and a visible note rather than a hard failure. + useEffect(() => { + let cancelled = false; + listResearchStudies().then((response) => { + if (cancelled) return; + if (response.ok) { + setStudies(Array.isArray(response.data) ? response.data : []); + setStudiesMissing(false); + setStudiesForbidden(false); + } else { + setStudies([]); + setStudiesMissing(!!response.missing); + setStudiesForbidden(!!response.forbidden); + } + }); + return () => { + cancelled = true; + }; + }, []); + + const loadStudy = useCallback(async (id) => { + const trimmed = (id || "").trim(); + if (!trimmed) { + setError("Enter a study ID first."); + return; + } + setIsLoading(true); + setError(""); + setNotice(""); + setValidationErrors([]); + setAppliedStudyId(trimmed); + setSelectedRevisionId(""); + setExposures([]); + setEnrollmentCoverage(null); + setCoverageError(""); + + const [revisionsRes, draftsRes] = await Promise.all([ + listResearchRevisions(trimmed), + listResearchDrafts(trimmed), + ]); + + if (revisionsRes.ok) { + setRevisions(Array.isArray(revisionsRes.data) ? revisionsRes.data : []); + } else { + setRevisions([]); + setError(revisionsRes.error); + } + if (draftsRes.ok) { + setDrafts(Array.isArray(draftsRes.data) ? draftsRes.data : []); + } else { + setDrafts([]); + if (revisionsRes.ok) setError(draftsRes.error); + } + setIsLoading(false); + }, []); + + useEffect(() => { + const fromUrl = searchParams.get("study_id"); + if (fromUrl) loadStudy(fromUrl); + // Only auto-load once from the initial URL; subsequent navigations use the + // form so we do not fight the researcher's edits. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Derived assignment/exposure counts for the selected revision. These come + // from researcher read models; a failure is shown as "unavailable" rather + // than as zero. + useEffect(() => { + if (!appliedStudyId || !selectedRevisionId) return undefined; + let cancelled = false; + (async () => { + const [exposuresRes, coverageRes] = await Promise.all([ + getResearchExposures(appliedStudyId, selectedRevisionId), + getResearchEnrollmentCoverage(appliedStudyId, selectedRevisionId), + ]); + if (cancelled) return; + setExposures(exposuresRes.ok ? exposuresRes.data : []); + setEnrollmentCoverage(coverageRes.ok ? coverageRes.data : null); + setCoverageError( + !exposuresRes.ok && !coverageRes.ok + ? exposuresRes.error || coverageRes.error + : !exposuresRes.ok + ? exposuresRes.error + : !coverageRes.ok + ? coverageRes.error + : "", + ); + })(); + return () => { + cancelled = true; + }; + }, [appliedStudyId, selectedRevisionId]); + + const handleLoad = (event) => { + event.preventDefault(); + if (!UUID_RE.test(studyId.trim())) { + setError("A study ID must be a UUID."); + return; + } + loadStudy(studyId); + }; + + const openNewDraft = () => { + navigate( + `/research/studies/${encodeURIComponent(appliedStudyId)}/editor`, + ); + }; + + const handleCreateStudy = async (event) => { + event.preventDefault(); + const name = newStudyName.trim(); + if (!name) { + setError("A study name is required."); + return; + } + setIsBusy(true); + setError(""); + setNotice(""); + setValidationErrors([]); + const description = newStudyDescription.trim(); + const result = await createResearchStudy({ + name, + description: description || null, + }); + if (result.ok) { + const study = result.data.study || {}; + setNewStudyName(""); + setNewStudyDescription(""); + setIsCreatingStudy(false); + const query = new URLSearchParams({ name }); + if (description) query.set("description", description); + navigate( + `/research/studies/${encodeURIComponent(study.study_id)}/editor?${query.toString()}`, + ); + } else { + setError(result.error); + } + setIsBusy(false); + }; + + const openDraft = (draftId) => { + navigate( + `/research/studies/${encodeURIComponent(appliedStudyId)}/editor?draft_id=${encodeURIComponent(draftId)}`, + ); + }; + + const supersedeRevision = (revisionId) => { + navigate( + `/research/studies/${encodeURIComponent(appliedStudyId)}/editor?revision_id=${encodeURIComponent(revisionId)}`, + ); + }; + + const handleValidateDraft = async (draftId) => { + setIsBusy(true); + setError(""); + setNotice(""); + setValidationErrors([]); + const draftRes = await getResearchDraft(draftId); + if (!draftRes.ok) { + setError(draftRes.error); + setIsBusy(false); + return; + } + const validation = await validateResearchProtocol(draftRes.protocol); + if (validation.ok && validation.valid) { + setNotice("Draft is valid and publishable."); + } else { + setValidationErrors(validation.errors || []); + setError(validation.error || "Draft failed validation."); + } + setIsBusy(false); + }; + + const handlePublishDraft = async (draftId) => { + setIsBusy(true); + setError(""); + setNotice(""); + setValidationErrors([]); + const draftRes = await getResearchDraft(draftId); + if (!draftRes.ok) { + setError(draftRes.error); + setIsBusy(false); + return; + } + const result = await publishResearchDraft(draftId, { + protocol: draftRes.protocol, + }); + if (result.ok) { + const revision = result.data.revision; + setNotice( + `Published revision #${revision ? revision.revision_number : "?"} (${ + revision ? shortId(revision.protocol_digest) : "" + }).`, + ); + await loadStudy(appliedStudyId); + } else if (result.status === 409) { + setError("Publication conflict: a newer revision exists. Reload and retry."); + } else { + setValidationErrors(result.errors || []); + setError(result.error); + } + setIsBusy(false); + }; + + const handleRetire = async (revision) => { + const confirmed = window.confirm( + `Retire revision #${revision.revision_number}? Its stored content and digest are unchanged; retirement is a lifecycle status only.`, + ); + if (!confirmed) return; + setIsBusy(true); + setError(""); + setNotice(""); + const result = await retireResearchRevision(revision.revision_id); + if (result.ok) { + setNotice( + result.data.retired + ? `Revision #${revision.revision_number} retired.` + : result.data.message || "Revision was already retired.", + ); + await loadStudy(appliedStudyId); + } else { + setError(result.error); + } + setIsBusy(false); + }; + + return ( +
+
+
+

Research Studies

+

+ Draft, validate, publish and supersede versioned study protocols. + Published revisions are immutable and content-addressed: editing one + creates a successor revision rather than mutating it. +

+
+
+ + +
+
+ + {isCreatingStudy && ( +
+

New study

+

+ The server mints the study UUID; you are taken straight to its + editor. No identifier needs to be copied and pasted. +

+ +